From ee9e550a484fda286eeab43b7db5f314b8b2d0d9 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 7 Sep 2026 13:57:09 +0000 Subject: [PATCH 001/144] feat(git-sync): sync extra_perms for variables (#11004) * feat(git-sync): sync extra_perms for variables Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Hvv5B8VP5Di4dbcCiVyZyE * refactor: trim the variable ACL-sync comment to the 4-line limit Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Hvv5B8VP5Di4dbcCiVyZyE * test: cover the revoke direction of variable extra_perms sync Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Hvv5B8VP5Di4dbcCiVyZyE --------- Co-authored-by: Claude Opus 5 (1M context) --- .../windmill-api-groups/src/granular_acls.rs | 26 ++++ backend/windmill-api/src/workspaces_export.rs | 11 +- cli/src/commands/sync/pull.ts | 2 +- cli/src/commands/variable/variable.ts | 72 +++++++---- cli/test/variable_resource_push.test.ts | 120 ++++++++++++++++++ 5 files changed, 197 insertions(+), 34 deletions(-) diff --git a/backend/windmill-api-groups/src/granular_acls.rs b/backend/windmill-api-groups/src/granular_acls.rs index c6f88dace5..07b8004d83 100644 --- a/backend/windmill-api-groups/src/granular_acls.rs +++ b/backend/windmill-api-groups/src/granular_acls.rs @@ -318,6 +318,19 @@ async fn add_granular_acl( ) .await? } + "variable" => { + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + DeployedObject::Variable { path: path.to_string(), parent_path: None }, + Some(format!("Variable '{}' changed permissions", path)), + true, + None, + ) + .await? + } _ => (), } @@ -528,6 +541,19 @@ async fn remove_granular_acl( ) .await? } + "variable" => { + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + DeployedObject::Variable { path: path.to_string(), parent_path: None }, + Some(format!("Variable '{}' changed permissions", path)), + true, + None, + ) + .await? + } _ => (), } } diff --git a/backend/windmill-api/src/workspaces_export.rs b/backend/windmill-api/src/workspaces_export.rs index 05cd354cd4..1c22981ee0 100644 --- a/backend/windmill-api/src/workspaces_export.rs +++ b/backend/windmill-api/src/workspaces_export.rs @@ -346,7 +346,7 @@ pub(crate) struct ArchiveQueryParams { default_ts: Option, /// Settings format version: "v1" (default) returns legacy flat format, "v2" returns grouped format settings_version: Option, - /// Opt-in: include `extra_perms` on flow / script / app rows. Default `false` + /// Opt-in: include `extra_perms` on script / flow / app / variable rows. Default `false` /// so cross-workspace tarball imports do not carry over ACLs referring to /// identities that may not exist in the target workspace. `wmill sync pull` /// passes `true` to surface ACLs in the git-tracked yaml. @@ -365,8 +365,8 @@ pub(crate) struct ArchiveQueryParams { /// pre-existing serialization for folders and groups so /// no customer sees a one-time noisy diff on upgrade. /// * `KeepIfNonEmpty` — keep when there is at least one entry, drop when `{}` -/// or null. New surface for flow / script / app, which -/// never carried ACLs in source before this change. +/// or null. New surface for script / flow / app / variable, +/// which never carried ACLs in source before this change. #[derive(Clone, Copy)] pub enum ExtraPermsBehavior { Drop, @@ -665,7 +665,7 @@ pub(crate) async fn tarball_workspace( check_scopes(&authed, || "variables:read".to_string())?; } - // Opt-in behavior for surfacing per-resource ACLs on flow/app rows. + // Opt-in behavior for surfacing per-resource ACLs on script/flow/app/variable rows. // Folder and group rows have always carried `extra_perms` in source and // continue to do so unconditionally (`KeepEvenEmpty`) so existing // customer git repos see no one-time noisy diff. @@ -1002,8 +1002,7 @@ pub(crate) async fn tarball_workspace( Error::internal_err(format!("Error decrypting variable {}: {}", var.path, e)) })?); } - let var_str = - &to_string_without_metadata(&var, ExtraPermsBehavior::Drop, None).unwrap(); + let var_str = &to_string_without_metadata(&var, new_kinds_extra_perms, None).unwrap(); archive .write_to_archive(&var_str, &format!("{}.variable.json", var.path)) .await?; diff --git a/cli/src/commands/sync/pull.ts b/cli/src/commands/sync/pull.ts index 2702e13714..68fd002eb5 100644 --- a/cli/src/commands/sync/pull.ts +++ b/cli/src/commands/sync/pull.ts @@ -104,7 +104,7 @@ export async function downloadZip( // from v1 the on-behalf-of address is stripped below, so the tarball sends the // `has_on_behalf_of` marker instead and never resolves an address. // `preserve_extra_perms=true` opts the tarball into surfacing granular ACLs - // on flow / script / app rows. Default-off on the server protects cross- + // on script / flow / app / variable rows. Default-off on the server protects cross- // workspace tarball imports from carrying ACLs that reference identities // missing in the target workspace; the CLI sync flow explicitly wants them. const baseParams = `&plain_secret=${plainSecrets ?? false diff --git a/cli/src/commands/variable/variable.ts b/cli/src/commands/variable/variable.ts index 2d885c9dd5..8963f376d2 100644 --- a/cli/src/commands/variable/variable.ts +++ b/cli/src/commands/variable/variable.ts @@ -19,6 +19,7 @@ import { sep as SEP } from "node:path"; import * as wmill from "../../../gen/services.gen.ts"; import { ListableVariable } from "../../../gen/types.gen.ts"; +import { applyExtraPermsDiff } from "../../core/extra_perms.ts"; async function list(opts: GlobalOptions & { json?: boolean }) { if (opts.json) log.setSilent(true); @@ -97,6 +98,7 @@ export interface VariableFile { description: string; account?: number; is_oauth?: boolean; + extra_perms?: Record; } /** @@ -152,37 +154,42 @@ export async function pushVariable( log.debug(`Variable ${remotePath} does not exist on remote`); } + // extra_perms is synced independently via /acls/* (see applyExtraPermsDiff) + // so a perm-only edit never rewrites the variable value. Strip the field from + // the body that goes to update_variable / create_variable and treat it as a + // separate step both for the up-to-date short-circuit and after the write. + const { extra_perms: localPerms, ...localVariableBody } = localVariable; + if (variable) { - if (isSuperset(localVariable, variable)) { + if (isSuperset(localVariableBody, variable)) { log.debug(`Variable ${remotePath} is up-to-date`); - return; - } + } else { + log.debug(`Variable ${remotePath} is not up-to-date, updating`); - log.debug(`Variable ${remotePath} is not up-to-date, updating`); - - // Apply is_secret only when it differs from the remote (the value is always - // sent, so the server allows the flag change). Upgrades (non-secret->secret) - // always apply; downgrades only when explicitly allowed (single-file push) — - // see allowSecretDowngrade. `undefined` leaves the flag untouched. - let nextIsSecret: boolean | undefined = undefined; - if (localVariable.is_secret !== variable.is_secret) { - if (localVariable.is_secret) { - nextIsSecret = true; - } else if (allowSecretDowngrade) { - nextIsSecret = false; + // Apply is_secret only when it differs from the remote (the value is always + // sent, so the server allows the flag change). Upgrades (non-secret->secret) + // always apply; downgrades only when explicitly allowed (single-file push) — + // see allowSecretDowngrade. `undefined` leaves the flag untouched. + let nextIsSecret: boolean | undefined = undefined; + if (localVariableBody.is_secret !== variable.is_secret) { + if (localVariableBody.is_secret) { + nextIsSecret = true; + } else if (allowSecretDowngrade) { + nextIsSecret = false; + } } - } - await wmill.updateVariable({ - workspace, - path: remotePath.replaceAll(SEP, "/"), - alreadyEncrypted: !plainSecrets, - requestBody: { - ...localVariable, - is_secret: nextIsSecret, - ...(wsSpecific !== undefined ? { ws_specific: wsSpecific } : {}), - }, - }); + await wmill.updateVariable({ + workspace, + path: remotePath.replaceAll(SEP, "/"), + alreadyEncrypted: !plainSecrets, + requestBody: { + ...localVariableBody, + is_secret: nextIsSecret, + ...(wsSpecific !== undefined ? { ws_specific: wsSpecific } : {}), + }, + }); + } } else { log.info(colors.yellow.bold(`Creating new variable ${remotePath}...`)); await wmill.createVariable({ @@ -190,11 +197,22 @@ export async function pushVariable( alreadyEncrypted: !plainSecrets, requestBody: { path: remotePath.replaceAll(SEP, "/"), - ...localVariable, + ...localVariableBody, ...(wsSpecific !== undefined ? { ws_specific: wsSpecific } : {}), }, }); } + + // Synced whether or not the body changed. No refetch: folder perms are never + // merged onto item.extra_perms, and the update/create body carries no + // extra_perms, so the value getVariable read above is still the remote one. + await applyExtraPermsDiff( + workspace, + "variable", + remotePath.replaceAll(SEP, "/"), + localPerms, + (variable as any)?.extra_perms, + ); } async function push( diff --git a/cli/test/variable_resource_push.test.ts b/cli/test/variable_resource_push.test.ts index e6bc84c326..b77ed48b50 100644 --- a/cli/test/variable_resource_push.test.ts +++ b/cli/test/variable_resource_push.test.ts @@ -361,6 +361,126 @@ describe("variable", () => { expect(content).toContain("is_secret: false"); }); }); + + test("extra_perms round-trips and pushes via /acls/* without rewriting the variable", async () => { + await withTestBackend(async (backend, tempDir) => { + await setupWorkspaceProfile(backend); + + const uniqueId = Date.now(); + const varPath = `f/test/perms_var_${uniqueId}`; + + const createResp = await backend.apiRequest!( + `/api/w/${backend.workspace}/variables/create`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + path: varPath, + value: "perms_test_value", + is_secret: false, + description: "Variable for extra_perms test", + }), + } + ); + expect(createResp.status).toBeLessThan(300); + await createResp.text(); + + const aclResp = await backend.apiRequest!( + `/api/w/${backend.workspace}/acls/add/variable/${varPath}`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ owner: "g/all", write: true }), + } + ); + expect(aclResp.status).toBeLessThan(300); + await aclResp.text(); + + await writeFile( + join(tempDir, "wmill.yaml"), + `defaultTs: bun\nincludes:\n - "${varPath}**"\nexcludes: []\n`, + "utf-8" + ); + + const pullResult = await backend.runCLICommand( + ["sync", "pull", "--yes"], + tempDir + ); + expect(pullResult.code).toEqual(0); + + const localPath = join(tempDir, `${varPath}.variable.yaml`); + const pulled = await readFile(localPath, "utf-8"); + expect(pulled).toContain("extra_perms:"); + expect(pulled).toContain("g/all: true"); + + const beforeResp = await backend.apiRequest!( + `/api/w/${backend.workspace}/variables/get/${varPath}` + ); + const before = await beforeResp.json(); + + // Perm-only edit: downgrade the grant to read. + await writeFile( + localPath, + pulled.replace("g/all: true", "g/all: false"), + "utf-8" + ); + + const pushResult = await backend.runCLICommand( + ["sync", "push", "--yes"], + tempDir + ); + expect(pushResult.code).toEqual(0); + + const afterResp = await backend.apiRequest!( + `/api/w/${backend.workspace}/variables/get/${varPath}` + ); + const after = await afterResp.json(); + expect(after.extra_perms).toEqual({ "g/all": false }); + // Routed through /acls/* rather than update_variable, so the row itself + // is untouched. + expect(after.edited_at).toEqual(before.edited_at); + expect(after.value).toEqual("perms_test_value"); + + // A yaml with no extra_perms field at all is "no opinion": a checkout + // that predates ACL sync must never revoke UI-managed grants. + await writeFile( + localPath, + `description: "Variable for extra_perms test"\nvalue: perms_test_value\nis_secret: false\n`, + "utf-8" + ); + const noOpinionResult = await backend.runCLICommand( + ["sync", "push", "--yes"], + tempDir + ); + expect(noOpinionResult.code).toEqual(0); + + const noOpinionApiResp = await backend.apiRequest!( + `/api/w/${backend.workspace}/variables/get/${varPath}` + ); + expect((await noOpinionApiResp.json()).extra_perms).toEqual({ + "g/all": false, + }); + + // An owner present remotely but absent from a *present* map is revoked — + // the one direction that can destroy a grant. + await writeFile( + localPath, + `description: "Variable for extra_perms test"\nvalue: perms_test_value\nis_secret: false\nextra_perms: {}\n`, + "utf-8" + ); + const revokeResult = await backend.runCLICommand( + ["sync", "push", "--yes"], + tempDir + ); + expect(revokeResult.code).toEqual(0); + + const finalResp = await backend.apiRequest!( + `/api/w/${backend.workspace}/variables/get/${varPath}` + ); + const final = await finalResp.json(); + expect(final.extra_perms).toEqual({}); + }); + }); }); // ============================================================================= From f381acdb37f66f5e272bc37938e69f734987d53f Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 7 Sep 2026 14:11:58 +0000 Subject: [PATCH 002/144] fix: seed runs page filter defaults through the url so they survive sync (#11005) Claude-Session: https://claude.ai/code/session_017KKZCLrTrWAqSeGjTzVtP2 Co-authored-by: Claude Opus 5 (1M context) --- .../src/lib/components/FilterSearchbar.svelte | 36 ++++++++++++++++-- frontend/src/lib/components/RunsPage.svelte | 37 +++++++++++-------- 2 files changed, 54 insertions(+), 19 deletions(-) diff --git a/frontend/src/lib/components/FilterSearchbar.svelte b/frontend/src/lib/components/FilterSearchbar.svelte index 51026f10fd..6c89f1a039 100644 --- a/frontend/src/lib/components/FilterSearchbar.svelte +++ b/frontend/src/lib/components/FilterSearchbar.svelte @@ -112,17 +112,37 @@ } /** - * Creates a URL-synced filter instance that automatically syncs with URL search parameters + * Creates a URL-synced filter instance that automatically syncs with URL search parameters. + * + * `initial` supplies defaults for keys the URL doesn't carry (a route segment, a persisted + * toggle); the returned `seed` re-applies them after a navigation has rewritten the query. */ export function useUrlSyncedFilterInstance( - schemaRec: T - ): { val: Partial> } { + schemaRec: T, + initial?: Partial> + ): { + val: Partial> + seed: (values: Partial>) => void + } { // Build the Zod schema from the filter schema const zodSchema = filterSchemaRecToZodSchema(schemaRec) // Create URL-synced search params const urlFilter = useSearchParams(zodSchema) as Record + // A default has to arrive as a URL param: the URL→instance effect below drops whatever the + // URL lacks, so a value written to the instance is undone on the next sync. Going through + // urlFilter rather than straight to history keeps the search-param cells in step, so it + // does not matter whether a popstate follows. + function seed(values: Partial>) { + const sp = new URLSearchParams(window.location.search) + for (const [key, value] of Object.entries(values) as [string, unknown][]) { + if (value === undefined || value === null || sp.has(key)) continue + urlFilter[key] = value instanceof Date ? value.toISOString() : value + } + } + if (initial) seed(initial) + // Create the filter instance object const filterInstance: { val: Partial> } = $state({ val: {} }) @@ -173,7 +193,15 @@ }) } - return filterInstance + return { + get val() { + return filterInstance.val + }, + set val(v: Partial>) { + filterInstance.val = v + }, + seed + } } function filterToText(filter: FilterInstance, schema: F): string { diff --git a/frontend/src/lib/components/RunsPage.svelte b/frontend/src/lib/components/RunsPage.svelte index 03b410d232..cac8952706 100644 --- a/frontend/src/lib/components/RunsPage.svelte +++ b/frontend/src/lib/components/RunsPage.svelte @@ -84,6 +84,8 @@ initialPath?: string } + let { initialPath }: Props = $props() + let paths: string[] = $state([]) let usernames: string[] = $state([]) let folders: string[] = $state([]) @@ -100,25 +102,30 @@ let perPage = useLocalStorageValue('runs_per_page', 1000, 'number') let showSchedulesStorage = useLocalStorageValue('runs_show_schedules', true, 'boolean') let showFutureJobsStorage = useLocalStorageValue('runs_show_future_jobs', true, 'boolean') - let filters = useUrlSyncedFilterInstance(untrack(() => runsFilterSearchbarSchema)) + function filterSeeds() { + return { + path: initialPath || undefined, + job_trigger_kind: showSchedulesStorage.val === false ? ('!schedule' as const) : undefined, + show_future_jobs: showFutureJobsStorage.val === false ? false : undefined + } + } - let { initialPath }: Props = $props() + let filters = useUrlSyncedFilterInstance( + untrack(() => runsFilterSearchbarSchema), + untrack(filterSeeds) + ) + + // `runs/[...path]` is a single route, so a navigation between its URLs — the sidebar's own + // "Runs" entry, `/runs/` → `/runs`, Back — rewrites the query without remounting, and + // what was seeded at mount is gone. Re-apply it. Editing a filter writes with `replaceState`, + // which never reaches `page.url`, so a filter the user clears stays cleared. + $effect(() => { + page.url.href + untrack(() => filters.seed(filterSeeds())) + }) let batchRerunOptionsIsOpen = $state(false) - // Initialize path filter from route param if provided and not already set via query params - if (untrack(() => initialPath) && !filters.val.path) { - filters.val.path = untrack(() => initialPath) - } - - // Apply persistent toggle values from local storage if URL doesn't specify them - if (!page.url.searchParams.has('job_trigger_kind') && showSchedulesStorage.val === false) { - filters.val.job_trigger_kind = '!schedule' - } - if (!page.url.searchParams.has('show_future_jobs') && showFutureJobsStorage.val === false) { - filters.val.show_future_jobs = false - } - // Sync toggle state back to local storage when filters change $effect(() => { if (!filters.val.job_trigger_kind || filters.val.job_trigger_kind === '!schedule') { From 7643e9bd77c56f72596b8dca50801baf58984198 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 7 Sep 2026 14:18:39 +0000 Subject: [PATCH 003/144] fix(cli): say which workspace id is targeted, and when wmill.yaml is bypassed (#11006) * fix(cli): say which workspace id is targeted and when wmill.yaml is bypassed Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01QaQ3UtkbHA6pxqQStqRQQj * fix(cli): make the wmill.yaml lookup for diagnostics side-effect free Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01QaQ3UtkbHA6pxqQStqRQQj * fix(cli): only report a wmill.yaml mapping that sets an explicit workspaceId Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01QaQ3UtkbHA6pxqQStqRQQj --------- Co-authored-by: Claude Opus 5 (1M context) --- cli/src/commands/sync/pull.ts | 13 +++- cli/src/core/conf.ts | 42 +++++++++++- cli/src/core/context.ts | 54 ++++++++++++++- ...base_url_workspace_resolution_unit.test.ts | 68 +++++++++++++++++++ 4 files changed, 173 insertions(+), 4 deletions(-) create mode 100644 cli/test/base_url_workspace_resolution_unit.test.ts diff --git a/cli/src/commands/sync/pull.ts b/cli/src/commands/sync/pull.ts index 68fd002eb5..d75dc81912 100644 --- a/cli/src/commands/sync/pull.ts +++ b/cli/src/commands/sync/pull.ts @@ -150,7 +150,18 @@ export async function downloadZip( } if (zipResponse.status === 404 || body.includes("no rows returned")) { - log.info(colors.red(`Workspace '${workspace.workspaceId}' not found on ${workspace.remote}. Please check your --workspace and try again.`)); + log.info( + colors.red( + `Workspace id '${workspace.workspaceId}' not found on ${workspace.remote}` + + (workspace.name !== workspace.workspaceId + ? ` (resolved from profile '${workspace.name}')` + : "") + + `.\n` + + `Note this is the workspace *id* sent to the API, which is not necessarily what you passed to --workspace:\n` + + ` - check 'wmill workspace list' (the 'workspace id' column)\n` + + ` - check the 'workspaces' block of wmill.yaml ('workspaceId' overrides the workspace name)` + ) + ); } else { log.info(colors.red(`Failed to request tarball from API: ${zipResponse.status} ${zipResponse.statusText}`)); if (body) log.info(colors.red(body)); diff --git a/cli/src/core/conf.ts b/cli/src/core/conf.ts index 1fcce7a43f..786ef48cc2 100644 --- a/cli/src/core/conf.ts +++ b/cli/src/core/conf.ts @@ -145,7 +145,9 @@ function getGitRepoRoot(): string | null { } export const GLOBAL_CONFIG_OPT = { noCdToRoot: false }; -function findWmillYaml(): string | null { + +// Pure upward search: no chdir, no logging. findWmillYaml() adds the chdir. +function locateWmillYaml(): string | null { const startDir = resolve(process.cwd()); const isInGitRepo = isGitRepository(); const gitRoot = isInGitRepo ? getGitRepoRoot() : null; @@ -176,6 +178,13 @@ function findWmillYaml(): string | null { currentDir = parentDir; } + return foundPath; +} + +function findWmillYaml(): string | null { + const startDir = resolve(process.cwd()); + const foundPath = locateWmillYaml(); + // If wmill.yaml was found in a parent directory, warn the user and change working directory if ( !GLOBAL_CONFIG_OPT.noCdToRoot && @@ -198,6 +207,37 @@ export function getWmillYamlPath(): string | null { return findWmillYaml(); } +/** + * Look up one `workspaces` entry, for diagnostics only. readConfigFile() must + * not be used for that: it chdirs to the config's directory, exits on an + * unsupported syncBehavior and throws on a malformed file. A diagnostic may + * never fail or relocate the command it is diagnosing. + */ +export async function peekWorkspaceEntry( + workspaceName: string +): Promise { + if (RESERVED_WORKSPACE_KEYS.has(workspaceName)) { + return undefined; + } + const wmillYamlPath = locateWmillYaml(); + if (!wmillYamlPath) { + return undefined; + } + try { + const conf = (await yamlParseFile(wmillYamlPath)) as SyncOptions; + const workspaces = + conf?.workspaces ?? + conf?.gitBranches ?? + conf?.environments ?? + conf?.git_branches; + const entry = (workspaces as any)?.[workspaceName]; + return typeof entry === "object" && entry !== null ? entry : undefined; + } catch (e) { + log.debug(`Failed to parse ${wmillYamlPath} for workspace lookup: ${e}`); + return undefined; + } +} + let legacyConfigWarned = false; export async function readConfigFile(opts?: { warnIfMissing?: boolean }): Promise { diff --git a/cli/src/core/context.ts b/cli/src/core/context.ts index 502157f0a7..80b27ea52e 100644 --- a/cli/src/core/context.ts +++ b/cli/src/core/context.ts @@ -20,6 +20,7 @@ import { import { getLastUsedProfile, setLastUsedProfile } from "./branch-profiles.ts"; import { readConfigFile, + peekWorkspaceEntry, findWorkspaceByGitBranch, getEffectiveWorkspaceId, getWmillYamlPath, @@ -219,6 +220,9 @@ async function tryResolveWorkspace( // First try: look up workspace by name in wmill.yaml workspaces config const config = await readConfigFile({ warnIfMissing: false }); const wsEntry = config.workspaces?.[opts.workspace] as WorkspaceEntryConfig | undefined; + // What wmill.yaml said to target, kept for the fallback below: a profile + // found by name can silently point somewhere else entirely. + let configuredTarget: { workspaceId: string; baseUrl: string } | undefined; if (wsEntry?.baseUrl) { const workspaceId = getEffectiveWorkspaceId(opts.workspace, wsEntry); let normalizedBaseUrl: string; @@ -231,6 +235,8 @@ async function tryResolveWorkspace( }; } + configuredTarget = { workspaceId, baseUrl: normalizedBaseUrl }; + // Find matching profile by baseUrl + workspaceId const allProfs = await allWorkspaces(opts.configDir); const matching = allProfs.filter( @@ -283,6 +289,22 @@ async function tryResolveWorkspace( ), }; } + if ( + configuredTarget && + (e.workspaceId !== configuredTarget.workspaceId || + e.remote !== configuredTarget.baseUrl) + ) { + log.warnStderr( + colors.yellow( + `⚠️ Falling back to the local profile named '${opts.workspace}' (${e.workspaceId} on ${e.remote}), which does NOT match wmill.yaml:\n` + + ` wmill.yaml maps workspace '${opts.workspace}' to ${configuredTarget.workspaceId} on ${configuredTarget.baseUrl}, but no profile targets it.\n` + + ` Run: wmill workspace add ${configuredTarget.workspaceId} ${configuredTarget.baseUrl}` + ) + ); + } + log.infoStderr( + `Using local profile '${e.name}' → ${e.workspaceId} on ${e.remote}` + ); (opts as any).__secret_workspace = e; return { isError: false, value: e }; } @@ -486,6 +508,8 @@ export async function resolveWorkspace( return process.exit(-1); } + let resolved: Workspace | undefined; + // Try to find existing workspace profile by name, then by workspaceId + remote if (opts.workspace) { let existingWorkspace = await getWorkspaceByName( @@ -523,19 +547,45 @@ export async function resolveWorkspace( ); return process.exit(-1); } - return { + resolved = { ...existingWorkspace, token: opts.token, }; } } - return { + resolved ??= { remote: normalizedBaseUrl, workspaceId: opts.workspace, name: opts.workspace, token: opts.token, }; + + // --base-url pins the target, so wmill.yaml's `workspaces` block is never + // consulted and `--workspace` reaches the API as a workspace id. Name the + // id being sent, and the mapping being skipped, before the request 404s + // on an id the user never typed. + // Only an explicit `workspaceId:` is worth reporting: an entry without one + // maps the name to itself, leaving nothing to correct. + const yamlEntry = await peekWorkspaceEntry(opts.workspace); + const yamlWorkspaceId = yamlEntry?.workspaceId; + if (yamlWorkspaceId && yamlWorkspaceId !== resolved.workspaceId) { + log.warnStderr( + colors.yellow( + `⚠️ --base-url is set, so wmill.yaml is not consulted: workspace id '${resolved.workspaceId}' is sent to the API.\n` + + ` wmill.yaml maps workspace '${opts.workspace}' to workspace id '${yamlWorkspaceId}'${yamlEntry!.baseUrl ? ` on ${yamlEntry!.baseUrl}` : ""}.\n` + + ` Use '--workspace ${yamlWorkspaceId}', or drop --base-url/--token to resolve through wmill.yaml.` + ) + ); + } + log.infoStderr( + `Using workspace id '${resolved.workspaceId}' on ${normalizedBaseUrl} (--base-url given` + + (resolved.name !== resolved.workspaceId + ? `, profile '${resolved.name}')` + : ")") + ); + (opts as any).__secret_workspace = resolved; + return resolved; } else { log.infoStderr( colors.red( diff --git a/cli/test/base_url_workspace_resolution_unit.test.ts b/cli/test/base_url_workspace_resolution_unit.test.ts new file mode 100644 index 0000000000..66f618d113 --- /dev/null +++ b/cli/test/base_url_workspace_resolution_unit.test.ts @@ -0,0 +1,68 @@ +import { describe, expect, test } from "bun:test"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; + +import { resolveWorkspace } from "../src/core/context.ts"; +import { getWorkspaceConfigFilePath } from "../windmill-utils-internal/src/config/config.ts"; +import type { GlobalOptions } from "../src/types.ts"; + +const BASE_URL = "http://localhost:9999/"; + +// --base-url pins the target: --workspace reaches the API as a workspace id and +// wmill.yaml is not consulted. The warning that says so may only peek at the +// file — readConfigFile() exits on an unsupported syncBehavior and throws on a +// malformed one, so resolving through it lets an unrelated config fail a +// command that never needed it. +async function withWmillYaml( + wmillYaml: string, + fn: (opts: GlobalOptions) => Promise +): Promise { + const repoDir = await mkdtemp(path.join(os.tmpdir(), "wmill_baseurl_repo_")); + const configDir = await mkdtemp(path.join(os.tmpdir(), "wmill_baseurl_conf_")); + const originalCwd = process.cwd(); + try { + await writeFile(path.join(repoDir, "wmill.yaml"), wmillYaml); + await writeFile(await getWorkspaceConfigFilePath(configDir), ""); + + process.chdir(repoDir); + await fn({ + configDir, + baseUrl: BASE_URL, + token: "sometoken", + workspace: "staging", + } as GlobalOptions); + } finally { + process.chdir(originalCwd); + await rm(repoDir, { recursive: true, force: true }); + await rm(configDir, { recursive: true, force: true }); + } +} + +describe("--base-url workspace resolution", () => { + const rejectedConfigs: [string, string][] = [ + ["an unsupported syncBehavior", "syncBehavior: v2\n"], + ["a malformed file", 'workspaces:\n staging:\n baseUrl: "unterminated\n'], + ]; + + for (const [label, wmillYaml] of rejectedConfigs) { + test(`resolves despite ${label}`, async () => { + await withWmillYaml(wmillYaml, async (opts) => { + const workspace = await resolveWorkspace(opts); + expect(workspace.workspaceId).toBe("staging"); + expect(workspace.remote).toBe(BASE_URL); + }); + }); + } + + test("a workspaces mapping never overrides the explicit workspace id", async () => { + await withWmillYaml( + "workspaces:\n staging:\n baseUrl: http://elsewhere.example/\n workspaceId: admins\n", + async (opts) => { + const workspace = await resolveWorkspace(opts); + expect(workspace.workspaceId).toBe("staging"); + expect(workspace.remote).toBe(BASE_URL); + } + ); + }); +}); From 5da4ea43fbd01e43aa14e75dc597d7ce5d8797ab Mon Sep 17 00:00:00 2001 From: AlexRV12 <71396855+AlexRV12@users.noreply.github.com> Date: Mon, 7 Sep 2026 16:22:09 +0200 Subject: [PATCH 004/144] feat: show the new-tab icon on a chat path pill while the modifier is held (#10976) * feat: show the new-tab icon on a chat path pill while the modifier is held Fixes WIN-2477 Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01LAa9nNcYxDN3qAZPrYg4Lq * fix: read the new-tab modifier in the capture phase Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01LAa9nNcYxDN3qAZPrYg4Lq * refactor: track the new-tab modifier only while a pill is hovered The window key listeners were installed at import time and never removed, so every page that loaded the module paid for them whether or not a pill existed. They now attach on mouseenter and detach on mouseleave or destroy, which is the only window in which the answer is read. Seeding the flag from the hover event also removes the limitation the previous version documented: a mouse event carries the same modifier flags as a key event, so a modifier held before the pointer arrived, or while this window was unfocused, now reads correctly. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01LAa9nNcYxDN3qAZPrYg4Lq * refactor: export the new-tab modifier as a read-only view `newTabModifier` handed every consumer a writable handle on module-global state, so any of them could drive the icon of every pill on the page. The getter form is what frontend/AGENTS.md prescribes for shared reactive state. Tearing each attachment down in the test's afterEach as well: the module state and its window listeners outlive the DOM, so emptying the body left `held` and the hovered node set for the following case. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01LAa9nNcYxDN3qAZPrYg4Lq * refactor: only track the modifier for pills whose icon can change The attachment went on every path pill, so hovering a drawer or plain-link pill installed three window listeners for a flag its icon never reads. Only a preview pill can flip, so only it gets them. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01LAa9nNcYxDN3qAZPrYg4Lq * fix: re-read the new-tab modifier from pointer movement A modifier held across a keyboard app switch was cleared by the blur and never restored: the key was down the whole time so no keydown arrived on the way back, and the pointer parked on the pill fired no fresh mouseenter either. The pill then showed the panel icon while the click would have opened a tab. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01LAa9nNcYxDN3qAZPrYg4Lq * refactor: give each pill its own modifier state The shared module state forced a node-identity guard: one hovered element owned the window listeners, so a pill destroyed elsewhere in the transcript had to be stopped from tearing them down. A factory per pill removes the guard, its test case, and the whole class of cross-instance interference, and narrows re-renders to the hovered pill instead of every preview pill on screen. Listener teardown now goes through AbortController signals, so leaving a pill drops the whole set at once rather than through a remove list that has to mirror every option exactly. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01LAa9nNcYxDN3qAZPrYg4Lq * fix: abort the previous hover controller on re-entry A second mouseenter with no mouseleave between replaced the controller without aborting it, so the four listeners registered under the first signal outlived even the element's destruction: neither leave nor the destroy path held a reference to reach them. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01LAa9nNcYxDN3qAZPrYg4Lq --------- Co-authored-by: Claude Opus 5 (1M context) --- .../attachments/newTabModifier.dom.test.ts | 106 ++++++++++++++++++ .../lib/attachments/newTabModifier.svelte.ts | 66 +++++++++++ .../copilot/chat/LinkRenderer.svelte | 13 ++- 3 files changed, 183 insertions(+), 2 deletions(-) create mode 100644 frontend/src/lib/attachments/newTabModifier.dom.test.ts create mode 100644 frontend/src/lib/attachments/newTabModifier.svelte.ts diff --git a/frontend/src/lib/attachments/newTabModifier.dom.test.ts b/frontend/src/lib/attachments/newTabModifier.dom.test.ts new file mode 100644 index 0000000000..115877298c --- /dev/null +++ b/frontend/src/lib/attachments/newTabModifier.dom.test.ts @@ -0,0 +1,106 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { newTabModifier } from './newTabModifier.svelte' + +const onPlatform = (userAgent: string) => vi.stubGlobal('navigator', { userAgent }) +const LINUX = 'Mozilla/5.0 (X11; Linux x86_64)' +const MAC = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)' + +const attached: (() => void)[] = [] + +/** Attach to a fresh element and return it with its cleanup, as `{@attach}` would. */ +function pill() { + const node = document.createElement('span') + document.body.append(node) + const modifier = newTabModifier() + const cleanup = modifier.attach(node) as () => void + attached.push(cleanup) + const hover = (init: MouseEventInit = {}) => + node.dispatchEvent(new MouseEvent('mouseenter', init)) + const move = (init: MouseEventInit = {}) => node.dispatchEvent(new MouseEvent('mousemove', init)) + const unhover = () => node.dispatchEvent(new MouseEvent('mouseleave')) + return { modifier, hover, move, unhover, cleanup } +} + +const keydown = (init: KeyboardEventInit) => + window.dispatchEvent(new KeyboardEvent('keydown', init)) + +describe('newTabModifier', () => { + // The window listeners outlive the DOM, so every case has to be torn down through the + // attachment rather than by emptying the body. + afterEach(() => { + attached.splice(0).forEach((cleanup) => cleanup()) + document.body.replaceChildren() + vi.unstubAllGlobals() + }) + + // The hover event carries the live modifier state, so a modifier pressed before the pointer + // arrived (or while this window was unfocused) is picked up rather than read as false. + it('seeds from the hover event, per platform', () => { + onPlatform(LINUX) + const linux = pill() + linux.hover({ ctrlKey: true }) + expect(linux.modifier.held).toBe(true) + + onPlatform(MAC) + const mac = pill() + // macOS ctrl+click is a secondary click, so it must not read as a new-tab modifier. + mac.hover({ ctrlKey: true }) + expect(mac.modifier.held).toBe(false) + mac.hover({ metaKey: true }) + expect(mac.modifier.held).toBe(true) + }) + + // Editors and menus stop keydown propagation to keep their own shortcuts, so a bubble-phase + // listener would go blind whenever focus sits in one. + it('sees a keydown that a focused element stops from propagating', () => { + onPlatform(LINUX) + const { modifier, hover } = pill() + hover() + const input = document.createElement('input') + input.addEventListener('keydown', (e) => e.stopPropagation()) + document.body.append(input) + + input.dispatchEvent( + new KeyboardEvent('keydown', { key: 'Control', ctrlKey: true, bubbles: true }) + ) + expect(modifier.held).toBe(true) + }) + + // A modifier held across a keyboard app switch is cleared by the blur and delivers no keydown + // on the way back, while the pointer parked on the pill fires no fresh mouseenter either. + it('re-seeds from pointer movement after the window lost focus', () => { + onPlatform(LINUX) + const { modifier, hover, move } = pill() + hover({ ctrlKey: true }) + window.dispatchEvent(new Event('blur')) + expect(modifier.held).toBe(false) + + move({ ctrlKey: true }) + expect(modifier.held).toBe(true) + }) + + it('stops tracking once unhovered', () => { + onPlatform(LINUX) + const { modifier, hover, unhover } = pill() + hover({ ctrlKey: true }) + unhover() + expect(modifier.held).toBe(false) + + keydown({ key: 'Control', ctrlKey: true }) + expect(modifier.held).toBe(false) + }) + + it('stops tracking when the element is destroyed while hovered', () => { + onPlatform(LINUX) + const { modifier, hover, cleanup } = pill() + hover({ ctrlKey: true }) + // Hovering again without leaving must not strand the first hover's listeners, which nothing + // would then hold a reference to. + hover({ ctrlKey: true }) + cleanup() + expect(modifier.held).toBe(false) + + keydown({ key: 'Control', ctrlKey: true }) + expect(modifier.held).toBe(false) + }) +}) diff --git a/frontend/src/lib/attachments/newTabModifier.svelte.ts b/frontend/src/lib/attachments/newTabModifier.svelte.ts new file mode 100644 index 0000000000..06c69f36cc --- /dev/null +++ b/frontend/src/lib/attachments/newTabModifier.svelte.ts @@ -0,0 +1,66 @@ +import type { Attachment } from 'svelte/attachments' +import { isMac } from '$lib/utils' + +/** + * Tracks whether the modifier that turns a click into a new browser tab is held, but only while + * the attached element is hovered, which is the only moment the answer is used. + */ +export function newTabModifier() { + let held = $state(false) + + // Only the modifier that actually yields a tab: shift opens a window, alt can start a + // download, and on macOS ctrl+click is a secondary click. + // Taken from each event rather than accumulated across keydown/keyup pairs, so a keyup lost to + // a focus change cannot strand the flag on. + const sync = (event: KeyboardEvent | MouseEvent) => { + held = isMac() ? event.metaKey : event.ctrlKey + } + const clear = () => { + held = false + } + + const attach: Attachment = (node) => { + // One controller per hover: a mirrored remove list leaks any listener whose options drift. + let hover: AbortController | undefined + const leave = () => { + hover?.abort() + hover = undefined + clear() + } + const enter = (event: MouseEvent) => { + // Seeded from the hover itself: mouse events carry the same modifier flags as key events, + // so a modifier already held before the pointer arrived reads correctly. + sync(event) + // Re-entering without an intervening leave would strand the previous controller: nothing + // else references it, so its listeners could never be removed. + hover?.abort() + hover = new AbortController() + const { signal } = hover + // Same reason the hover seeds: a modifier held across a keyboard app switch delivers no + // keydown on the way back, so the pointer is all that is left to re-read it from. + node.addEventListener('mousemove', sync, { signal }) + // Capture: editors and menus stopPropagation the keys they handle, hiding the modifier + // from a bubble-phase listener whenever focus sits in one. + window.addEventListener('keydown', sync, { capture: true, signal }) + window.addEventListener('keyup', sync, { capture: true, signal }) + // Not capture, unlike the two above: blur does not bubble but does reach the window while + // capturing, so it would fire for every element that loses focus. + window.addEventListener('blur', clear, { signal }) + } + + const life = new AbortController() + node.addEventListener('mouseenter', enter, { signal: life.signal }) + node.addEventListener('mouseleave', leave, { signal: life.signal }) + return () => { + life.abort() + leave() + } + } + + return { + get held() { + return held + }, + attach + } +} diff --git a/frontend/src/lib/components/copilot/chat/LinkRenderer.svelte b/frontend/src/lib/components/copilot/chat/LinkRenderer.svelte index 1e885fa304..e93017b095 100644 --- a/frontend/src/lib/components/copilot/chat/LinkRenderer.svelte +++ b/frontend/src/lib/components/copilot/chat/LinkRenderer.svelte @@ -3,6 +3,7 @@ import { ExternalLink, PanelRight } from 'lucide-svelte' import { Button } from '$lib/components/common' import RowIcon from '$lib/components/common/table/RowIcon.svelte' + import { newTabModifier } from '$lib/attachments/newTabModifier.svelte' import { hasToolDisplayActionHandler, runToolDisplayAction @@ -44,6 +45,8 @@ const previewAction = $derived(available?.type === 'open_item_preview' ? available : undefined) const drawerAction = $derived(available?.type === 'open_created_resource' ? available : undefined) + const modifier = newTabModifier() + const hint = $derived( previewAction ? `Open ${wmPath} in the preview panel` : `Open ${wmPath} in a new tab` ) @@ -67,7 +70,11 @@ {#if href} {#if wmKind} - + + - {#if previewAction} + + {#if previewAction && !modifier.held} {:else} From e2b63d177ae4e5c980cb5da34154540c90771b63 Mon Sep 17 00:00:00 2001 From: Diego Imbert <70353967+diegoimbert@users.noreply.github.com> Date: Mon, 7 Sep 2026 16:46:00 +0200 Subject: [PATCH 005/144] feat: go to referenced row from foreign-keyed cells in the database manager (#10998) * feat: go to referenced row from foreign-keyed cells in the database manager Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_0157Kw1ukbQo7eZnmyM63G4t * fix: pin foreign keys to their table and escape backslashes on snowflake Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_0157Kw1ukbQo7eZnmyM63G4t * fix: address review on foreign key navigation (stale fetch, qualifiers, chip) Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_0157Kw1ukbQo7eZnmyM63G4t * fix: unicode literals on sql server and hide unreachable foreign key targets Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_0157Kw1ukbQo7eZnmyM63G4t --------- Co-authored-by: Claude Fable 5.1 --- frontend/src/lib/components/DBManager.svelte | 107 ++++++++++-- .../lib/components/DBManagerContent.svelte | 3 +- frontend/src/lib/components/DBTable.svelte | 160 +++++++++++++++--- .../lib/components/DbForeignKeyTooltip.svelte | 23 +++ .../display/dbtable/renderDbLiteral.test.ts | 48 ++++++ .../apps/components/display/dbtable/utils.ts | 51 +++++- frontend/src/lib/components/dbOps.ts | 84 +++++---- 7 files changed, 394 insertions(+), 82 deletions(-) create mode 100644 frontend/src/lib/components/DbForeignKeyTooltip.svelte create mode 100644 frontend/src/lib/components/apps/components/display/dbtable/renderDbLiteral.test.ts diff --git a/frontend/src/lib/components/DBManager.svelte b/frontend/src/lib/components/DBManager.svelte index 35bd88abec..7520cc5013 100644 --- a/frontend/src/lib/components/DBManager.svelte +++ b/frontend/src/lib/components/DBManager.svelte @@ -12,8 +12,8 @@ import { Pane, Splitpanes } from 'svelte-splitpanes' import { ClearableInput, Drawer, DrawerContent } from './common' import { sendUserToast } from '$lib/toast' - import { type ColumnDef } from './apps/components/display/dbtable/utils' - import DBTable from './DBTable.svelte' + import { renderDbEqualityFilter, type ColumnDef } from './apps/components/display/dbtable/utils' + import DBTable, { type DbForeignKeyTarget, type DbRowFilter } from './DBTable.svelte' import type { IDbSchemaOps, IDbTableOps } from './dbOps' import DropdownV2 from './DropdownV2.svelte' import ConfirmationModal from './common/confirmationModal/ConfirmationModal.svelte' @@ -46,7 +46,12 @@ dbSupportsSchemas: boolean databaseIsEmpty?: boolean colDefs: Record | undefined - dbTableOpsFactory: (params: { colDefs: ColumnDef[]; tableKey: string }) => IDbTableOps + dbTableOpsFactory: (params: { + colDefs: ColumnDef[] + tableKey: string + /** Raw SQL predicate AND-ed into the reads (already escaped). */ + whereClause?: string + }) => IDbTableOps dbSchemaOps: IDbSchemaOps refresh?: () => void initialSchemaKey?: string @@ -214,6 +219,83 @@ : selected.tableKey ) + // Set by "Go to row" on a foreign-keyed cell; pinned to the table it was + // created for so a schema change can't carry it onto an unrelated table. + let rowFilter: (DbRowFilter & { tableKey: string }) | undefined = $state() + let activeRowFilter = $derived(rowFilter?.tableKey === tableKey ? rowFilter : undefined) + let whereClause = $derived( + activeRowFilter + ? renderDbEqualityFilter(activeRowFilter.column, activeRowFilter.value, dbType) + : undefined + ) + + function selectTable(schemaKey: string | undefined, table: string) { + rowFilter = undefined + selected = { schemaKey, tableKey: table } + } + + /** Where a foreign key's `schema.table` target lives in the sidebar, or + * undefined when it cannot be opened from here. */ + function resolveForeignKeyTarget( + targetTable: string + ): { schemaKey: string; table: string } | undefined { + const parts = targetTable.split('.') + const table = parts[parts.length - 1] + const qualifier = parts.length > 1 ? parts.slice(0, -1).join('.') : undefined + // Without schema support the sidebar browses the connection's default + // schema only, and unqualified reads would hit a same-named local table. + if (!dbSupportsSchemas && qualifier && qualifier !== selected.schemaKey) return undefined + const schemaKey = dbSupportsSchemas && qualifier ? qualifier : selected.schemaKey + if (!schemaKey || !(table in (dbSchema.schema[schemaKey] ?? {}))) return undefined + return { schemaKey, table } + } + + function goToRow(target: DbForeignKeyTarget) { + const resolved = resolveForeignKeyTarget(target.table) + if (!resolved) { + sendUserToast(`Table ${target.table} cannot be opened from this schema`, true) + return + } + if (renderDbEqualityFilter(target.column, target.value, dbType) === undefined) { + sendUserToast('This value cannot be used as a filter', true) + return + } + const { schemaKey, table } = resolved + selectTable(schemaKey, table) + rowFilter = { + tableKey: dbSupportsSchemas ? `${schemaKey}.${table}` : table, + column: target.column, + value: target.value + } + } + + // The result carries the table it was fetched for: `resource` keeps the + // previous value while refetching, and a stale list would decorate the new + // table's same-named columns as foreign keys. + let foreignKeys = resource( + [() => selected.tableKey, () => selected.schemaKey, () => colDefs], + async ([table, schema], _prev, { signal }) => { + if (!table) return undefined + const forTableKey = dbSupportsSchemas && schema ? `${schema}.${table}` : table + const fks = + features?.foreignKeys === false + ? [] + : await dbSchemaOps.onFetchForeignKeys({ table, schema }) + // A newer selection started meanwhile: an AbortError keeps this result + // out of `current`, where it would shadow the newer table's keys. + if (signal.aborted) throw new DOMException('Superseded', 'AbortError') + return { tableKey: forTableKey, foreignKeys: fks } + } + ) + // Only keys whose target the sidebar can open get the "Go to row" affordance. + let currentForeignKeys = $derived.by(() => { + const fetched = foreignKeys.current + if (!fetched || fetched.tableKey !== tableKey) return undefined + return fetched.foreignKeys.filter( + (fk) => fk.targetTable && resolveForeignKeyTarget(fk.targetTable) !== undefined + ) + }) + let askingForConfirmation: | (ConfirmationModal['$$prop_def'] & { onConfirm: () => void }) | undefined = $state() @@ -395,14 +477,12 @@ role="button" tabindex="0" onclick={() => { - selected.schemaKey = schemaKey - selected.tableKey = tableKey + selectTable(schemaKey, tableKey) toggleTableSelection(schemaKey, tableKey) }} onkeydown={(e) => { if (e.key === 'Enter' || e.key === ' ') { - selected.schemaKey = schemaKey - selected.tableKey = tableKey + selectTable(schemaKey, tableKey) toggleTableSelection(schemaKey, tableKey) } }} @@ -468,7 +548,7 @@ + diff --git a/frontend/src/lib/components/apps/components/display/dbtable/renderDbLiteral.test.ts b/frontend/src/lib/components/apps/components/display/dbtable/renderDbLiteral.test.ts new file mode 100644 index 0000000000..1dec5e9da8 --- /dev/null +++ b/frontend/src/lib/components/apps/components/display/dbtable/renderDbLiteral.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from 'vitest' +import { renderDbEqualityFilter, renderDbLiteral } from './utils' + +describe('renderDbLiteral', () => { + it('doubles single quotes on every dialect', () => { + expect(renderDbLiteral("O'Brien", 'postgresql')).toBe("'O''Brien'") + expect(renderDbLiteral("O'Brien", 'mysql')).toBe("'O''Brien'") + }) + + it('doubles backslashes only where the dialect treats them as escapes', () => { + expect(renderDbLiteral('C:\\dir\\', 'postgresql')).toBe("'C:\\dir\\'") + expect(renderDbLiteral('C:\\dir\\', 'mysql')).toBe("'C:\\\\dir\\\\'") + expect(renderDbLiteral('C:\\dir\\', 'snowflake')).toBe("'C:\\\\dir\\\\'") + }) + + it('marks SQL Server strings as Unicode constants', () => { + expect(renderDbLiteral("Zoë's", 'ms_sql_server')).toBe("N'Zoë''s'") + }) + + it('renders numbers and booleans without quotes', () => { + expect(renderDbLiteral(42, 'postgresql')).toBe('42') + expect(renderDbLiteral(true, 'postgresql')).toBe('TRUE') + expect(renderDbLiteral(true, 'ms_sql_server')).toBe('1') + }) + + it('has no literal for values that cannot be compared safely', () => { + expect(renderDbLiteral(null, 'postgresql')).toBeUndefined() + expect(renderDbLiteral({ a: 1 }, 'postgresql')).toBeUndefined() + expect(renderDbLiteral(NaN, 'postgresql')).toBeUndefined() + }) +}) + +describe('renderDbEqualityFilter', () => { + it('quotes the identifier per dialect', () => { + expect(renderDbEqualityFilter('user id', 'x', 'postgresql')).toBe(`"user id" = 'x'`) + expect(renderDbEqualityFilter('user id', 'x', 'ms_sql_server')).toBe(`[user id] = N'x'`) + expect(renderDbEqualityFilter('user id', 'x', 'mysql')).toBe("`user id` = 'x'") + expect(renderDbEqualityFilter('user id', null, 'postgresql')).toBeUndefined() + }) + + it('doubles a delimiter embedded in the identifier', () => { + expect(renderDbEqualityFilter('a"b', 1, 'postgresql')).toBe(`"a""b" = 1`) + expect(renderDbEqualityFilter('a"b', 1, 'snowflake')).toBe(`"a""b" = 1`) + expect(renderDbEqualityFilter('a"b', 1, 'duckdb')).toBe(`"a""b" = 1`) + expect(renderDbEqualityFilter('a]b', 1, 'ms_sql_server')).toBe(`[a]]b] = 1`) + expect(renderDbEqualityFilter('a`b', 1, 'mysql')).toBe('`a``b` = 1') + }) +}) diff --git a/frontend/src/lib/components/apps/components/display/dbtable/utils.ts b/frontend/src/lib/components/apps/components/display/dbtable/utils.ts index bf81ec07bb..c0477e8408 100644 --- a/frontend/src/lib/components/apps/components/display/dbtable/utils.ts +++ b/frontend/src/lib/components/apps/components/display/dbtable/utils.ts @@ -333,25 +333,58 @@ export function duckdbQuicksearchColumns(columnDefs: ColumnDef[]): string { .join(', ') } +/** Mirrors the backend's `render_db_quoted_identifier`, including doubling an + * embedded delimiter. */ export function renderDbQuotedIdentifier(identifier: string, dbType: DbType): string { switch (dbType) { case 'postgresql': - return `"${identifier}"` // PostgreSQL uses double quotes for identifiers - case 'ms_sql_server': - return `[${identifier}]` // MSSQL uses square brackets for identifiers - case 'mysql': - return `\`${identifier}\`` // MySQL uses backticks case 'snowflake': - return `"${identifier}"` // Snowflake uses double quotes for identifiers - case 'bigquery': - return `\`${identifier}\`` // BigQuery uses backticks case 'duckdb': - return `"${identifier}"` // DuckDB uses double quotes for identifiers + return `"${identifier.replace(/"/g, '""')}"` + case 'ms_sql_server': + return `[${identifier.replace(/]/g, ']]')}]` + case 'mysql': + case 'bigquery': + return `\`${identifier.replace(/`/g, '``')}\`` default: throw new Error('Unsupported database type: ' + dbType) } } +/** Renders a cell value as a SQL literal. Returns undefined for values that + * have no safe literal form (null, objects, non-finite numbers). */ +export function renderDbLiteral(value: unknown, dbType: DbType): string | undefined { + if (value === null || value === undefined) return undefined + if (typeof value === 'number') return Number.isFinite(value) ? String(value) : undefined + if (typeof value === 'bigint') return value.toString() + if (typeof value === 'boolean') { + if (dbType === 'ms_sql_server') return value ? '1' : '0' + return value ? 'TRUE' : 'FALSE' + } + if (typeof value !== 'string') return undefined + let escaped = value.replace(/'/g, "''") + // MySQL, Snowflake and BigQuery treat a backslash inside a string literal as + // an escape character. + if (dbType === 'mysql' || dbType === 'snowflake' || dbType === 'bigquery') { + escaped = escaped.replace(/\\/g, '\\\\') + } + // A plain constant is varchar on SQL Server and goes through the database + // code page; the N prefix keeps it Unicode against nvarchar columns. + return dbType === 'ms_sql_server' ? `N'${escaped}'` : `'${escaped}'` +} + +/** `"column" = ` predicate, or undefined when the value can't be + * rendered as a literal. */ +export function renderDbEqualityFilter( + column: string, + value: unknown, + dbType: DbType +): string | undefined { + const literal = renderDbLiteral(value, dbType) + if (literal === undefined) return undefined + return `${renderDbQuotedIdentifier(column, dbType)} = ${literal}` +} + export function getLanguageByResourceType(name: string): ScriptLang { const language = { postgresql: 'postgresql', diff --git a/frontend/src/lib/components/dbOps.ts b/frontend/src/lib/components/dbOps.ts index a89abf4374..5005cc931c 100644 --- a/frontend/src/lib/components/dbOps.ts +++ b/frontend/src/lib/components/dbOps.ts @@ -14,6 +14,7 @@ import { WorkspaceService } from '$lib/gen' import { pendingMigrations } from './workspaceSettings/datatableMigrationUtils' import { buildTableEditorValues, + type TableEditorForeignKey, type TableEditorValues } from './apps/components/display/dbtable/tableEditor' import { type AlterTableValues } from './apps/components/display/dbtable/queries/alterTable' @@ -250,6 +251,10 @@ export type IDbSchemaOps = { schema?: string colDefs: TableMetadata }) => Promise + onFetchForeignKeys: (params: { + table: string + schema?: string + }) => Promise } /** Thrown by a schema op when the user declines the out-of-order run warning. @@ -396,6 +401,48 @@ export function dbSchemaOpsWithPreviewScripts({ } } + /** Resolves to [] when the database has no foreign key introspection + * (BigQuery) or the query fails: callers treat foreign keys as optional. */ + async function fetchForeignKeys({ + table, + schema + }: { + table: string + schema?: string + }): Promise { + if (dbType === 'bigquery') return [] + try { + const fkContent = makeMarker('FOREIGN_KEYS', { table, schema }) + const fkResult = await runScriptAndPollResult({ + workspace, + requestBody: { args: dbArg, content: fkContent, language, tag } + }) + + let rawForeignKeys: RawForeignKey[] + if (dbType === 'snowflake') { + rawForeignKeys = transformSnowflakeForeignKeys(fkResult as any[]) + } else { + rawForeignKeys = fkResult as RawForeignKey[] + if (rawForeignKeys && Array.isArray(rawForeignKeys)) { + rawForeignKeys = rawForeignKeys.map((fk) => { + const lowerFk: any = {} + Object.keys(fk).forEach((key) => { + lowerFk[key.toLowerCase()] = fk[key] + }) + return lowerFk + }) + } + } + + if (rawForeignKeys && Array.isArray(rawForeignKeys)) { + return transformForeignKeys(rawForeignKeys) + } + } catch (e) { + console.warn('Failed to fetch foreign keys:', e) + } + return [] + } + return { onDelete: async ({ tableKey, schema }) => { const content = makeMarker('DROP_TABLE', { table: tableKey, schema }) @@ -454,44 +501,11 @@ export function dbSchemaOpsWithPreviewScripts({ const downContent = makeMarker('CREATE_SCHEMA', { schema }) await applyDdl(migrationName('drop_schema', schema), content, downContent) }, + onFetchForeignKeys: fetchForeignKeys, onFetchTableEditorDefinition: async ({ table, schema, colDefs }) => { - let foreignKeys: import('./apps/components/display/dbtable/tableEditor').TableEditorForeignKey[] = - [] + const foreignKeys = await fetchForeignKeys({ table, schema }) let pk_constraint_name: string | undefined - // Fetch foreign keys (not supported for BigQuery) - if (dbType !== 'bigquery') { - try { - const fkContent = makeMarker('FOREIGN_KEYS', { table, schema }) - const fkResult = await runScriptAndPollResult({ - workspace, - requestBody: { args: dbArg, content: fkContent, language, tag } - }) - - let rawForeignKeys: RawForeignKey[] - if (dbType === 'snowflake') { - rawForeignKeys = transformSnowflakeForeignKeys(fkResult as any[]) - } else { - rawForeignKeys = fkResult as RawForeignKey[] - if (rawForeignKeys && Array.isArray(rawForeignKeys)) { - rawForeignKeys = rawForeignKeys.map((fk) => { - const lowerFk: any = {} - Object.keys(fk).forEach((key) => { - lowerFk[key.toLowerCase()] = fk[key] - }) - return lowerFk - }) - } - } - - if (rawForeignKeys && Array.isArray(rawForeignKeys)) { - foreignKeys = transformForeignKeys(rawForeignKeys) - } - } catch (e) { - console.warn('Failed to fetch foreign keys:', e) - } - } - // Fetch primary key constraint name (not supported for BigQuery/MySQL) if (dbType !== 'bigquery' && dbType !== 'mysql') { try { From 5f3f99ba6915b7c5df663a30b35f4cd02050e728 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Mon, 7 Sep 2026 16:46:35 +0200 Subject: [PATCH 006/144] fix(cli): keep permissioned_as on single-item push, as sync push does (#11000) * fix(cli): keep permissioned_as on single-item push, as sync push does * fix(cli): resolve syncBehavior from the target workspace, not the branch alone * refactor(cli): share the workspace-name resolution between sync and single-item push * test(cli): import the moved workspace-name helper from its new home --- cli/src/commands/app/app.ts | 20 ++- cli/src/commands/flow/flow.ts | 21 ++- cli/src/commands/schedule/schedule.ts | 19 ++- cli/src/commands/script/script.ts | 8 +- cli/src/commands/sync/sync.ts | 88 ++---------- cli/src/commands/trigger/trigger.ts | 11 +- cli/src/core/conf.ts | 77 +++++++++++ cli/src/core/permissioned_as.ts | 30 +++++ ...schedule_push_permissioned_as_unit.test.ts | 125 ++++++++++++++++++ ...workspace_key_filename_integration.test.ts | 6 +- 10 files changed, 316 insertions(+), 89 deletions(-) create mode 100644 cli/test/schedule_push_permissioned_as_unit.test.ts diff --git a/cli/src/commands/app/app.ts b/cli/src/commands/app/app.ts index 0f46ec76be..b35efc02a3 100644 --- a/cli/src/commands/app/app.ts +++ b/cli/src/commands/app/app.ts @@ -12,7 +12,11 @@ import * as wmill from "../../../gen/services.gen.ts"; import { ListableApp, Policy } from "../../../gen/types.gen.ts"; import { GlobalOptions, isSuperset } from "../../types.ts"; -import { getWmillYamlPath, mergeConfigWithConfigFile } from "../../core/conf.ts"; +import { + getWmillYamlPath, + mergeConfigWithConfigFile, + readEffectiveSyncBehavior, +} from "../../core/conf.ts"; import { readInlinePathSync } from "../../utils/utils.ts"; import devCommand from "./dev.ts"; import lintCommand from "./lint.ts"; @@ -21,6 +25,7 @@ import newCommand from "./new.ts"; import generateAgentsCommand from "./generate_agents.ts"; import { isVersionsGeq1585 } from "../sync/global.ts"; import type { PermissionedAsContext } from "../../core/permissioned_as.ts"; +import { buildPermissionedAsContext } from "../../core/permissioned_as.ts"; import { applyExtraPermsDiff } from "../../core/extra_perms.ts"; export interface AppFile { @@ -420,6 +425,8 @@ async function push( if (isRawAppByName || hasRawAppYaml) { const { pushRawApp } = await import("./raw_apps.ts"); const merged = await mergeConfigWithConfigFile(opts); + // Raw-app ownership preservation is not implemented on either push + // path: sync push hands pushRawApp no context either. await pushRawApp( workspace.workspaceId, remotePath, @@ -429,7 +436,16 @@ async function push( ); log.info(colors.bold.underline.green("Raw app pushed")); } else { - await pushApp(workspace.workspaceId, remotePath, absoluteFilePath); + await pushApp( + workspace.workspaceId, + remotePath, + absoluteFilePath, + undefined, + await buildPermissionedAsContext( + workspace.workspaceId, + await readEffectiveSyncBehavior(opts, workspace), + ), + ); log.info(colors.bold.underline.green("App pushed")); } } diff --git a/cli/src/commands/flow/flow.ts b/cli/src/commands/flow/flow.ts index 7a383fcfeb..863690ce7d 100644 --- a/cli/src/commands/flow/flow.ts +++ b/cli/src/commands/flow/flow.ts @@ -4,7 +4,7 @@ import { Command } from "@cliffy/command"; import { Confirm } from "@cliffy/prompt/confirm"; import { Table } from "@cliffy/table"; import * as log from "../../core/log.ts"; -import { dirname, sep as SEP } from "node:path"; +import { dirname, sep as SEP, resolve as pathResolve } from "node:path"; import { stringify as yamlStringify } from "yaml"; import { yamlParseFile } from "../../utils/yaml.ts"; import { readTextFile, validateRequiredArgs } from "../../utils/utils.ts"; @@ -21,11 +21,16 @@ import { } from "../../core/context.ts"; import { resolve, track_job, pollForJobResult } from "../script/script.ts"; import { defaultFlowDefinition } from "../../../bootstrap/flow_bootstrap.ts"; -import { SyncOptions, mergeConfigWithConfigFile } from "../../core/conf.ts"; +import { + SyncOptions, + mergeConfigWithConfigFile, + readEffectiveSyncBehavior, +} from "../../core/conf.ts"; import { FSFSElement, elementsToMap, ignoreF } from "../sync/sync.ts"; import { Flow } from "../../../gen/types.gen.ts"; import { applyExtraPermsDiff } from "../../core/extra_perms.ts"; import type { PermissionedAsContext } from "../../core/permissioned_as.ts"; +import { buildPermissionedAsContext } from "../../core/permissioned_as.ts"; import { collectPathScriptPaths, replaceInlineScripts, @@ -327,10 +332,20 @@ async function push(opts: Options & { message?: string }, filePath: string, remo if (!validatePath(remotePath)) { return; } + // Reading the config moves the cwd to the wmill.yaml root when it sits in a + // parent directory, so pin the file against the invocation cwd first. + filePath = pathResolve(filePath); const workspace = await resolveWorkspace(opts); await requireLogin(opts); + const syncBehavior = await readEffectiveSyncBehavior(opts, workspace); - await pushFlow(workspace.workspaceId, remotePath, filePath, opts.message); + await pushFlow( + workspace.workspaceId, + remotePath, + filePath, + opts.message, + await buildPermissionedAsContext(workspace.workspaceId, syncBehavior) + ); log.info(colors.bold.underline.green("Flow pushed")); } diff --git a/cli/src/commands/schedule/schedule.ts b/cli/src/commands/schedule/schedule.ts index 2eb6ba42b8..096615c757 100644 --- a/cli/src/commands/schedule/schedule.ts +++ b/cli/src/commands/schedule/schedule.ts @@ -6,13 +6,19 @@ import { Command } from "@cliffy/command"; import { Table } from "@cliffy/table"; import { colors } from "@cliffy/ansi/colors"; import * as log from "../../core/log.ts"; -import { sep as SEP } from "node:path"; +import { sep as SEP, resolve as pathResolve } from "node:path"; import { requireLogin } from "../../core/auth.ts"; import { resolveWorkspace, validatePath } from "../../core/context.ts"; -import { mergeConfigWithConfigFile } from "../../core/conf.ts"; +import { + mergeConfigWithConfigFile, + readEffectiveSyncBehavior, +} from "../../core/conf.ts"; import * as wmill from "../../../gen/services.gen.ts"; import type { PermissionedAsContext } from "../../core/permissioned_as.ts"; -import { lookupUsernameByEmail } from "../../core/permissioned_as.ts"; +import { + buildPermissionedAsContext, + lookupUsernameByEmail, +} from "../../core/permissioned_as.ts"; import { GlobalOptions, @@ -299,8 +305,12 @@ async function disable(opts: GlobalOptions, path: string) { } async function push(opts: GlobalOptions, filePath: string, remotePath: string) { + // Reading the config moves the cwd to the wmill.yaml root when it sits in a + // parent directory, so pin the file against the invocation cwd first. + filePath = pathResolve(filePath); const workspace = await resolveWorkspace(opts); await requireLogin(opts); + const syncBehavior = await readEffectiveSyncBehavior(opts, workspace); if (!validatePath(remotePath)) { return; @@ -317,7 +327,8 @@ async function push(opts: GlobalOptions, filePath: string, remotePath: string) { workspace.workspaceId, remotePath, undefined, - parseFromFile(filePath) + parseFromFile(filePath), + await buildPermissionedAsContext(workspace.workspaceId, syncBehavior) ); console.log(colors.bold.underline.green("Schedule pushed")); } diff --git a/cli/src/commands/script/script.ts b/cli/src/commands/script/script.ts index fada434e0d..d2afdeeee3 100644 --- a/cli/src/commands/script/script.ts +++ b/cli/src/commands/script/script.ts @@ -7,6 +7,7 @@ import { validatePath, } from "../../core/context.ts"; import type { PermissionedAsContext } from "../../core/permissioned_as.ts"; +import { buildPermissionedAsContext } from "../../core/permissioned_as.ts"; import { applyExtraPermsDiff } from "../../core/extra_perms.ts"; import { writeFile, stat, mkdir } from "node:fs/promises"; import { Buffer } from "node:buffer"; @@ -58,6 +59,7 @@ import { SyncOptions, mergeConfigWithConfigFile, readConfigFile, + readEffectiveSyncBehavior, } from "../../core/conf.ts"; import { SyncCodebase, listSyncCodebases } from "../../utils/codebase.ts"; import { pollJobWithQueueLogging } from "../../utils/job_polling.ts"; @@ -231,7 +233,11 @@ async function push(opts: PushOptions, filePath: string) { opts.message, opts, await getRawWorkspaceDependencies(true), - codebases + codebases, + await buildPermissionedAsContext( + workspace.workspaceId, + await readEffectiveSyncBehavior(opts, workspace) + ) ); log.info(colors.bold.underline.green(`Script ${filePath} pushed`)); } diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index be5d9e5639..99a098f848 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -76,7 +76,8 @@ import { } from "../../utils/utils.ts"; import { getEffectiveSettings, - getWorkspaceNames, + inferWsNameFromProfile, + resolveWsNameForConfigFromFlags, mergeConfigWithConfigFile, parseSyncBehavior, SyncOptions, @@ -85,7 +86,10 @@ import { WorkspaceEntryConfig, } from "../../core/conf.ts"; import type { PermissionedAsContext } from "../../core/permissioned_as.ts"; -import { preCheckPermissionedAs } from "../../core/permissioned_as.ts"; +import { + buildPermissionedAsContext, + preCheckPermissionedAs, +} from "../../core/permissioned_as.ts"; import { fromWorkspaceSpecificPath, toWorkspaceSpecificPath, @@ -429,37 +433,6 @@ export function computeWsSpecificFlagOnlyPushes( return out; } -// Resolve workspace name from a --branch override (git branch → workspace name). -// Falls back to using the branch value as-is (backward compat: old key = branch name). -function resolveWsNameFromBranch( - opts: SyncOptions, - branchName: string, -): string { - const match = findWorkspaceByGitBranch(opts.workspaces, branchName); - return match ? match[0] : branchName; -} - -// Resolve wsNameForConfig from CLI flags. Prefers --branch → matching config key, -// then --workspace → matching config key (incl. when --base-url is set). Returns -// undefined when no flag-based resolution applies; callers then fall back to -// inferWsNameFromProfile on the resolved workspace profile. -export function resolveWsNameForConfigFromFlags( - opts: SyncOptions & { branch?: string; workspace?: string }, -): string | undefined { - if (opts.branch) { - return resolveWsNameFromBranch(opts, opts.branch); - } - if (opts.workspace) { - // Use getWorkspaceNames so reserved keys (e.g. commonSpecificItems) are filtered out, - // matching the behavior of findWorkspaceByGitBranch / inferWsNameFromProfile. - const validKeys = getWorkspaceNames(opts.workspaces); - if (validKeys.includes(opts.workspace)) { - return opts.workspace; - } - } - return undefined; -} - // Warn if --workspace overrides auto-detected branch or if workspace not in config. function warnWorkspaceOverride( opts: SyncOptions, @@ -507,33 +480,6 @@ function resolveWsNameForFiles(_opts: SyncOptions, wsName: string): string { return wsName; } -// After resolveWorkspace, infer the workspace config name from the resolved profile -// by matching baseUrl + workspaceId against the workspaces config entries. -function inferWsNameFromProfile( - opts: SyncOptions, - profile: { remote: string; workspaceId: string }, -): string | undefined { - if (!opts.workspaces) return undefined; - const wsNames = Object.keys(opts.workspaces).filter( - (k) => k !== "commonSpecificItems", - ); - for (const name of wsNames) { - const entry = (opts.workspaces as any)[name] as WorkspaceEntryConfig; - if (!entry?.baseUrl) continue; - try { - const entryUrl = new URL(entry.baseUrl).toString(); - const profileUrl = new URL(profile.remote).toString(); - const entryWsId = entry.workspaceId ?? name; - if (entryUrl === profileUrl && entryWsId === profile.workspaceId) { - return name; - } - } catch { - continue; - } - } - return undefined; -} - // Merge CLI options with effective settings, preserving CLI flags as overrides function mergeCliWithEffectiveOptions< T extends GlobalOptions & SyncOptions & { repository?: string }, @@ -5540,27 +5486,19 @@ export async function push( return; } - let permissionedAsContext: PermissionedAsContext | undefined = undefined; - if (parseSyncBehavior(opts.syncBehavior) >= 1) { - const user = await wmill.whoami({ workspace: workspace.workspaceId }); - const userIsAdminOrDeployer = - user.is_admin || (user.groups ?? []).includes("wm_deployers"); - log.debug( - `permissioned_as: user=${user.email}, is_admin=${user.is_admin}, groups=${JSON.stringify(user.groups)}, isAdminOrDeployer=${userIsAdminOrDeployer}`, + const permissionedAsContext: PermissionedAsContext | undefined = + await buildPermissionedAsContext( + workspace.workspaceId, + opts.syncBehavior, ); - permissionedAsContext = { - userCache: new Map(), - userIsAdminOrDeployer, - userEmail: user.email, - }; - + if (permissionedAsContext) { // ws_specific_flag changes have no content payload, so they don't // affect permissioned_as resolution — filter them out before the // pre-check (which expects only added/edited/deleted). await preCheckPermissionedAs( changes.filter((c) => c.name !== "ws_specific_flag"), - user.email, - userIsAdminOrDeployer, + permissionedAsContext.userEmail, + permissionedAsContext.userIsAdminOrDeployer, opts.acceptOverridingPermissionedAsWithSelf ?? false, !!process.stdin.isTTY, ); diff --git a/cli/src/commands/trigger/trigger.ts b/cli/src/commands/trigger/trigger.ts index 83b62f9eb4..b011a0f0b7 100644 --- a/cli/src/commands/trigger/trigger.ts +++ b/cli/src/commands/trigger/trigger.ts @@ -23,7 +23,7 @@ import { Command } from "@cliffy/command"; import { Table } from "@cliffy/table"; import { colors } from "@cliffy/ansi/colors"; import * as log from "../../core/log.ts"; -import { sep as SEP } from "node:path"; +import { sep as SEP, resolve as pathResolve } from "node:path"; import { GlobalOptions, isSuperset, @@ -41,6 +41,8 @@ import { getCurrentGitBranch } from "../../utils/git.ts"; import { requireLogin } from "../../core/auth.ts"; import { validatePath, resolveWorkspace } from "../../core/context.ts"; import type { PermissionedAsContext } from "../../core/permissioned_as.ts"; +import { buildPermissionedAsContext } from "../../core/permissioned_as.ts"; +import { readEffectiveSyncBehavior } from "../../core/conf.ts"; type Trigger = { http: HttpTrigger; @@ -620,8 +622,12 @@ async function extractTriggerKindFromPath(filePath: string): Promise.overrides.syncBehavior`, which is where a repo + * that varies settings per workspace puts it, and the entry to read is the one + * `--workspace` names — falling back to the profile, then to the git branch — + * the same order `sync push` resolves it in. + */ +export async function readEffectiveSyncBehavior( + opts: { workspace?: string }, + profile?: { remote: string; workspaceId: string } +): Promise { + const config = await readConfigFile({ warnIfMissing: false }); + const named = resolveWsNameForConfigFromFlags({ ...config, ...opts }); + const effective = await getEffectiveSettings( + config, + undefined, + false, + true, + named ?? (profile ? inferWsNameFromProfile(config, profile) : undefined) + ); + return effective.syncBehavior; +} + const RESERVED_WORKSPACE_KEYS = new Set(["commonSpecificItems"]); /** diff --git a/cli/src/core/permissioned_as.ts b/cli/src/core/permissioned_as.ts index 5ac48753d6..57570b3549 100644 --- a/cli/src/core/permissioned_as.ts +++ b/cli/src/core/permissioned_as.ts @@ -3,6 +3,7 @@ import * as log from "./log.ts"; import { colors } from "@cliffy/ansi/colors"; import { Confirm } from "@cliffy/prompt/confirm"; import { getTypeStrFromPath } from "../types.ts"; +import { parseSyncBehavior } from "./conf.ts"; export interface PermissionedAsContext { userCache: Map; @@ -10,6 +11,35 @@ export interface PermissionedAsContext { userEmail: string; } +/** + * The whole-tree `sync push` and the single-item `push` commands must resolve + * ownership the same way, so both build the context here: a push that leaves it + * undefined reassigns `permissioned_as` / `on_behalf_of` to whoever ran it. + * Undefined below syncBehavior v1, where that reassignment is the contract, and + * for a caller who is neither admin nor in `wm_deployers` the backend enforces + * it anyway — the flag on the context is what keeps the CLI from claiming + * otherwise. + */ +export async function buildPermissionedAsContext( + workspace: string, + syncBehavior: string | number | undefined +): Promise { + if (parseSyncBehavior(syncBehavior) < 1) { + return undefined; + } + const user = await wmill.whoami({ workspace }); + const userIsAdminOrDeployer = + user.is_admin || (user.groups ?? []).includes("wm_deployers"); + log.debug( + `permissioned_as: user=${user.email}, is_admin=${user.is_admin}, groups=${JSON.stringify(user.groups)}, isAdminOrDeployer=${userIsAdminOrDeployer}` + ); + return { + userCache: new Map(), + userIsAdminOrDeployer, + userEmail: user.email, + }; +} + async function ensureUserCache( workspace: string, cache: Map diff --git a/cli/test/schedule_push_permissioned_as_unit.test.ts b/cli/test/schedule_push_permissioned_as_unit.test.ts new file mode 100644 index 0000000000..941415409d --- /dev/null +++ b/cli/test/schedule_push_permissioned_as_unit.test.ts @@ -0,0 +1,125 @@ +/** + * Regression guard: the standalone `wmill schedule push` must resolve ownership + * the same way `wmill sync push` does. It only preserves the remote's + * `permissioned_as` when the command hands `pushSchedule` a context, so a push + * that builds none silently reassigns the schedule to whoever ran it. + */ + +import { expect, test, describe, beforeEach, mock } from "bun:test"; +import { mkdtemp, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +let updateScheduleCalls: any[] = []; +let remotePermissionedAs: string | undefined = "u/svc"; + +const REMOTE_SCHEDULE = () => ({ + path: "u/admin/sched", + schedule: "0 0 */6 * * *", + timezone: "Etc/UTC", + script_path: "u/admin/script", + is_flow: false, + args: {}, + enabled: false, + summary: "before", + permissioned_as: remotePermissionedAs, +}); + +mock.module("../gen/services.gen.ts", () => ({ + getSchedule: async () => REMOTE_SCHEDULE(), + updateSchedule: async (a: unknown) => { + updateScheduleCalls.push(a); + }, + whoami: async () => ({ + email: "deployer@windmill.dev", + username: "deployer", + is_admin: true, + groups: [], + }), +})); + +const realContext = await import("../src/core/context.ts"); +mock.module("../src/core/context.ts", () => ({ + ...realContext, + resolveWorkspace: async () => ({ + workspaceId: "w", + name: "w", + remote: "http://localhost/", + token: "t", + }), +})); + +const realAuth = await import("../src/core/auth.ts"); +mock.module("../src/core/auth.ts", () => ({ + ...realAuth, + requireLogin: async () => ({}), +})); + +const scheduleCommand = (await import("../src/commands/schedule/schedule.ts")) + .default; + +async function pushIn(wmillYamlTail: string): Promise { + const dir = await mkdtemp(join(tmpdir(), "windmill_sched_push_")); + await writeFile( + join(dir, "wmill.yaml"), + `defaultTs: bun\nincludeSchedules: true\n${wmillYamlTail}`, + "utf-8" + ); + await writeFile( + join(dir, "sched.schedule.yaml"), + `schedule: "0 0 */6 * * *"\ntimezone: Etc/UTC\nscript_path: u/admin/script\nis_flow: false\nargs: {}\nenabled: false\nsummary: after\n`, + "utf-8" + ); + + const cwd = process.cwd(); + process.chdir(dir); + try { + await scheduleCommand.parse([ + "push", + "sched.schedule.yaml", + "u/admin/sched", + ]); + } finally { + process.chdir(cwd); + } +} + +describe("wmill schedule push ownership", () => { + beforeEach(() => { + updateScheduleCalls = []; + remotePermissionedAs = "u/svc"; + }); + + test("keeps the remote's permissioned_as under syncBehavior v1", async () => { + await pushIn("syncBehavior: v1\n"); + + expect(updateScheduleCalls).toHaveLength(1); + const body = updateScheduleCalls[0].requestBody; + expect(body.summary).toBe("after"); + expect(body.permissioned_as).toBe("u/svc"); + expect(body.preserve_permissioned_as).toBe(true); + }); + + // The entry to read is the one matching the workspace being pushed to, not + // the top level: a repo that varies settings per workspace puts syncBehavior + // under `overrides` and nowhere else. + test("reads syncBehavior from the target workspace's overrides", async () => { + await pushIn( + `workspaces:\n other:\n baseUrl: http://localhost/\n workspaceId: w\n overrides:\n syncBehavior: v1\n` + ); + + expect(updateScheduleCalls).toHaveLength(1); + const body = updateScheduleCalls[0].requestBody; + expect(body.permissioned_as).toBe("u/svc"); + expect(body.preserve_permissioned_as).toBe(true); + }); + + test("leaves ownership to the backend below syncBehavior v1", async () => { + await pushIn(""); + + expect(updateScheduleCalls).toHaveLength(1); + const body = updateScheduleCalls[0].requestBody; + expect(body.permissioned_as).toBeUndefined(); + expect(body.preserve_permissioned_as).toBeUndefined(); + }); +}); diff --git a/cli/test/workspace_key_filename_integration.test.ts b/cli/test/workspace_key_filename_integration.test.ts index ae22f191bf..a2e731eae5 100644 --- a/cli/test/workspace_key_filename_integration.test.ts +++ b/cli/test/workspace_key_filename_integration.test.ts @@ -7,8 +7,10 @@ import { stringify as yamlStringify } from "yaml"; import { resolveWsNameForGitBranch } from "../src/core/specific_items.ts"; import { findResourceFile } from "../src/commands/script/script.ts"; -import { resolveWsNameForConfigFromFlags } from "../src/commands/sync/sync.ts"; -import type { SyncOptions } from "../src/core/conf.ts"; +import { + resolveWsNameForConfigFromFlags, + type SyncOptions, +} from "../src/core/conf.ts"; // Integration tests covering the bug where workspace-specific filenames used // the raw git branch name instead of the wmill.yaml workspace config key. From c6e0302d7c1c60147f19d55a3923be8b1aa99c9c Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 7 Sep 2026 15:38:07 +0000 Subject: [PATCH 007/144] feat: let `// materialize` declare a `dbt://` warehouse-relation write (#10978) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: let `// materialize` declare a `dbt://` warehouse-relation write `// materialize manual dbt:////` lets an ingestion script in any language declare that it writes a warehouse relation, so it and the dbt model reading that relation land on one asset node instead of two disconnected pictures. `manual` is the only mode a warehouse target has — nothing generates warehouse DDL — and the non-`manual` spelling is refused rather than silently degraded. The `` segment is resolved against the workspace's configured warehouses, like a descriptor's `profile.warehouse`. The run records the same `materialized_partition` row a DuckLake target does, from the generic job path rather than an executor: the DuckLake write engine is DuckDB's, this declaration is anyone's. With a non-dbt producer now possible, the blanket deploy-time refusal of `# on dbt://` narrows to the shape that still cannot fire — every writer of the relation being a dbt script, since a dbt run does not dispatch. "Nothing produces it yet" stays accepted, as for every other asset kind, so deploy order does not matter. A dbt script may not subscribe at all: its graph ingest clears its own `dbt://` trigger rows. The one ordering the deploy cannot catch — a subscription accepted before any producer, then claimed by a dbt project — is named in that project's deploy log. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Rw1WrKeRRzyYHjfkuB83ek * fix: address review — preview stamping, stale producer set, public doc Three findings from the local review round: - Record the warehouse write only for a DEPLOYED script job. The annotation is a deploy-time contract (`manual`, three segments, a configured warehouse) checked where write access to the path is also required; honouring it in a preview, hub or inline-flow body let `jobs:run` alone restamp any relation's last writer from a script that never touched it. - Exclude the deploying script's own rows from the producer set. Read committed, they describe the version being replaced, so a script dropping its `// materialize` while adding a subscription counted itself as the producer that would wake it and committed a dormant edge. It could not be that producer anyway — the dispatcher skips self-loops. - `AssetKind::Dbt`'s doc no longer claims dbt is the exclusive producer of a warehouse relation, on both the types and the parser enum. Co-Authored-By: Claude Opus 5 (1M context) * fix: review round 1 — dbt-script materialize, set-form rule, doc - Refuse `// materialize` on a dbt script, the producer half of the rule the trigger loop already applies to `// on`: the graph ingest republishes that path's asset rows wholesale, so a declared write is wiped by the deploy that accepted it while its runs keep stamping the relation. - `dormant_dbt_subscriptions` now spells the same predicate its singular sibling does: the producer set has to be non-empty (nothing produces it yet is deploy order, not a dormant edge) and excludes the subscriber's own path (a script never wakes itself). Both divergences are pinned by tests. - The docs no longer claim the dbt deploy log covers a native producer that drops its `// materialize`; it does not, and nothing else reports that case. - An integration test over the deploy contract, since only a real deploy proves the handler feeds `sole_dbt_producer` the canonical key `asset.path` holds — the spelling that has to agree across the materialize target, the `// on` ref and the refusal that joins them. Co-Authored-By: Claude Opus 5 (1M context) * docs: qualify the any-language claim, and pin the dbt-script refusal `AssetKind::Dbt`'s contract (both enums), the two runtime guides and the deploy comment said a script of any language may declare a `dbt://` write, which the dbt-script refusal added last round contradicts. They now say "any language but dbt's own", with the reason: a project's writes are read from its manifest. The deploy-contract integration test covers that refusal for both annotations. Co-Authored-By: Claude Opus 5 (1M context) * docs: teach the pipeline AI guidance the warehouse-relation target The pipeline prompt (both sources, plus the regenerated bundle) told the model `// materialize` is DuckDB-only and rejected on any other target, which now steers users away from the very thing this PR adds. It distinguishes the managed DuckLake write, still DuckDB-only, from the warehouse-relation declaration any language but dbt's own may make. `dbt_manifest.rs`'s module doc carried the same "the only thing that creates one" overclaim the other four sites lost last commit. Co-Authored-By: Claude Opus 5 (1M context) * fix: draw an explicit dbt:// subscription on the canvas The editor suppressed every `// on dbt://…` overlay, which was right while the deploy refused all of them. It now refuses only a relation dbt alone builds, so the suppression hid the author's own annotation for exactly the case this PR adds — a subscription woken by a native `// materialize manual dbt://…` producer. The deploy stays the gate. Also the two stale claims round 4 named: the live pipeline prompt dropped the dbt-script exception the base prompt carries, and the doc's e2e requirements still said every `dbt://` subscription is refused. Co-Authored-By: Claude Opus 5 (1M context) * fix: refuse `// data_test` beside a `dbt://` materialize target `// data_test` checks are verifier probes the DuckDB executor splices around a managed write. A warehouse relation is written by the script itself, in any language, so nothing would run them — and unlike the DuckLake `manual` case, which at least fails loudly in that executor, a declarer in another language deployed green with its data-quality assertions silently skipped. Covered in the deploy-contract test and documented beside the annotation. Co-Authored-By: Claude Opus 5 (1M context) * fix: exclude a renamed producer from the sole-dbt producer set The producer set already excluded the deploying script's own path, because its committed rows describe the version being replaced. Under a rename the write sits at the OLD path — still committed, and removed by the same uncommitted transaction — so a producer renamed while it drops its `// materialize` and adds `// on dbt://…` still counted as the producer that would wake it, and committed a dormant edge. The deploy-contract test covers it: without the exclusion the rename deploys 201 instead of being refused. Co-Authored-By: Claude Opus 5 (1M context) * test: take the rename test's parent hash from the create response `format!("{:x}", …)` over the stored i64 drops leading zeros, while `ScriptHash`'s deserializer hex-decodes and demands 8 bytes — so a hash below 2^60 would 422 the request instead of reaching the refusal it asserts on, on roughly one in sixteen spellings of that script body. The create response already carries the zero-padded form, as the rest of the suite uses. Co-Authored-By: Claude Opus 5 (1M context) * docs: state the concurrent-ingest interleaving honestly `sole_dbt_producer`'s doc claimed the concurrent-deploy race only ever resolves toward refusing. It does when the uncommitted producer is native; when it is the dbt ingest, the check sees an empty producer set and accepts, and if that ingest then commits and runs its warning query before the subscriber's trigger row lands, neither side reports the dormant edge. Not serialized: the two would have to share a per-relation lock, and the ingest takes `script … FOR UPDATE` before its own advisory lock, so a deploy holding relation locks first inverts that order into a cross-subsystem deadlock — a worse failure than the cosmetic edge. Recorded beside the other orphaning the deploy cannot catch, with the bound both share: the next deploy of that project warns. Co-Authored-By: Claude Opus 5 (1M context) * fix: refuse a `dbt://` subscription that is not a whole relation `# on dbt://main/analytics` deployed and persisted a trigger row. Every producer spells `//` — the manifest ingest derives it from `relation_name`, a `// materialize` target is checked against it — so a partial one is an edge nothing can ever wake, which is what the dbt-only refusal exists to prevent. The shape now has one definition (`is_full_relation_path`) that both halves of the deploy ask, rather than a segment count spelled twice: a subscription and a write that disagreed would refuse and accept the same string. Also rewrites the canvas test's comment as a current constraint per AGENTS.md. Co-Authored-By: Claude Opus 5 (1M context) * fix: hold both halves of the deploy to one `dbt://` relation validator A subscription checked the relation's shape but not its warehouse, so `# on dbt:////` deployed and persisted a trigger row for something no producer can ever write: the write side refuses that exact string, and a dbt project's `profile.warehouse` resolves against the same config, so no later deploy fixes it and the dormant-edge warning cannot report it either. The shape rule and the warehouse rule now live in one `validate_dbt_relation` that both halves call, rather than being spelled per site — the previous two rounds each closed one half of one rule, which is the drift that invites. Also moves the parser test out from between a comment and the test it documents, and names both refusals in the doc's list. Co-Authored-By: Claude Opus 5 (1M context) * fix: drop the subscription-only clause from the shared refusal message "so nothing can produce it" reads backwards on the `// materialize` side, which is the producer. The remaining sentence says what is wrong on both. Co-Authored-By: Claude Opus 5 (1M context) * fix: bound a `dbt://` relation by the asset-path column in the shared validator `asset.path` is VARCHAR(255) and the manifest ingest drops a relation that outgrows it rather than failing the whole graph, so past the column no producer row can exist on either side. `script_trigger.trigger_ref` is unbounded text, so an overlong subscription deployed and stayed dormant for good; an overlong write reached Postgres and failed the deploy on a `value too long` instead of a message. Both now refuse in the validator the two halves share, against the ingest's own constant. The integration case computes the ref from that constant so it cannot drift back under the bound. Co-Authored-By: Claude Opus 5 (1M context) * fix: report a warehouse-lookup failure as the failure it is, and correct the boundary `dbt_warehouse_exists` fails three ways — no such warehouse, the query itself, and a setting with no `resource_path` — and all three became a 400 blaming the user's warehouse name. A pool timeout mid-deploy told a retrying sync that a transient server error was a permanent client one. Only `NotFound` is the annotation's fault now. The known-boundary paragraph claimed a flow-runner run still cascades. It does not: it is routed by `flow_step_id`, which `is_eligible_kind` rejects, as `asset_trigger_dispatch.rs` pins. Recording and cascading are decided separately, so the paragraph now names all three routes rather than merging two of them — and the row it omitted, an ordinary flow step, which records and never cascades. E2E item 7 said "deployable" where the rule is "wakeable": with only the dbt project reading the relation the producer set is empty, which deploys fine. Co-Authored-By: Claude Opus 5 (1M context) * docs: correct two rationales the last commit got wrong `Error::SqlErr` already maps to 400 in this codebase, so the query case's status was never the thing at stake. What the `NotFound` match earns is that a query failure and a malformed setting stop being described as an unconfigured warehouse name, and that the malformed-setting `InternalErr` reaches its own 500 instead of being flattened. And a flow step is two shapes, not one: a step running a deployed script is a `Script` job that records and never cascades, while a step with an inline body is `FlowScript`, which the recording guard excludes along with previews. Co-Authored-By: Claude Opus 5 (1M context) * fix: warn about dormant subscriptions from the run that publishes ownership too A run whose static descriptor finds its profile moved re-ingests the version's graph and republishes path ownership, exactly as a deploy does — so it can be what leaves a subscription accepted while the relation had no producer with dbt as its only one. That path discarded `persist_ingest`'s result and emitted no warning, which also made the doc's enumeration of unreported orphanings wrong. Both ownership-publishing points warn now. An agent worker still cannot: it reaches these tables only through the API and its ingest publishes without reading back, which the doc now says. Co-Authored-By: Claude Opus 5 (1M context) * docs: an agent run publishes no ownership, and the warning has two callers The agent-worker sentence called it an exception that publishes ownership without warning. It publishes none: `Connection::Http` forces per-run models, and `publishes_ownership()` is the negation of that, so an agent stores a job-pinned snapshot and leaves workspace ownership with the deployed graph — it cannot orphan a subscription at all. `warn_dormant_subscribers`' own doc still named the deploy log as the only place the warning shows, one commit after it gained its second caller. Co-Authored-By: Claude Opus 5 (1M context) * docs: stop the managed-write rule from contradicting the dbt:// target The sentence after the warehouse-relation paragraph says `// materialize` means the runtime writes the table for you and the body is a bare SELECT. That is the managed DuckLake rule, written before a `dbt://` target existed, and unqualified it tells the model the opposite of what the paragraph above it just said — a model following the more prominent one emits a SELECT for a warehouse relation, which deploys and then writes nothing. Both prompt sources now scope it, and both name the `// data_test` refusal beside a `dbt://` target, which the badge list advertised without the caveat. Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Opus 5 (1M context) --- ...84411c125cbfada861992e3d6032de635bad6.json | 12 + ...0cb1059f63c1ea51db3471c213018b2983704.json | 12 + ...7f518433741305d891d99059184eaf39fa2db.json | 20 ++ ...e613381cb4dcf7e851197150a3771ab43c99e.json | 23 ++ ...b9877190091923a2615d7d94bdf89a12e43c7.json | 12 + ...81b0a03e44beea80da732a4ad7166cf4c06cf.json | 20 ++ ...638a42f03599ce5eb872cf601f1da2e76b946.json | 63 +++++ ...3b3b6856b44d49c0712f08e307f950f210570.json | 12 + .../windmill-parser/src/asset_parser.rs | 69 +++-- .../tests/dbt_materialize_target.rs | 250 ++++++++++++++++++ backend/windmill-api-scripts/src/scripts.rs | 202 +++++++++++--- backend/windmill-common/src/assets.rs | 138 +++++++++- backend/windmill-common/src/dbt_manifest.rs | 8 +- .../tests/dbt_producer_rules.rs | 196 ++++++++++++++ backend/windmill-queue/src/asset_dispatch.rs | 12 +- backend/windmill-types/src/assets.rs | 23 +- backend/windmill-worker/src/dbt_executor.rs | 69 ++++- .../windmill-worker/src/duckdb_executor.rs | 4 +- backend/windmill-worker/src/worker.rs | 107 +++++++- docs/dbt-runtime.md | 150 ++++++++--- docs/ducklake-materialization.md | 6 +- .../assets/AssetGraph/resolveGraph.test.ts | 12 + .../assets/AssetGraph/resolveGraph.ts | 28 +- frontend/src/lib/components/assets/lib.ts | 9 +- .../components/copilot/chat/pipeline/core.ts | 4 +- system_prompts/auto-generated/prompts.ts | 8 +- system_prompts/base/pipeline-base.md | 8 +- 27 files changed, 1326 insertions(+), 151 deletions(-) create mode 100644 backend/.sqlx/query-1d8effff5dd1e4a177efed7366e84411c125cbfada861992e3d6032de635bad6.json create mode 100644 backend/.sqlx/query-1f7608defb5748db687145750520cb1059f63c1ea51db3471c213018b2983704.json create mode 100644 backend/.sqlx/query-33aa15957f56281947963cd696f7f518433741305d891d99059184eaf39fa2db.json create mode 100644 backend/.sqlx/query-49d905cae6ba42a3df62bad9385e613381cb4dcf7e851197150a3771ab43c99e.json create mode 100644 backend/.sqlx/query-6890e2be43ff8c653ce62f8cde1b9877190091923a2615d7d94bdf89a12e43c7.json create mode 100644 backend/.sqlx/query-ae4c0e8aeeef90d08a9b9c919b881b0a03e44beea80da732a4ad7166cf4c06cf.json create mode 100644 backend/.sqlx/query-b8161f6481460bed1c985bedc7c638a42f03599ce5eb872cf601f1da2e76b946.json create mode 100644 backend/.sqlx/query-e37b74f77cfa8769aee4d95155a3b3b6856b44d49c0712f08e307f950f210570.json create mode 100644 backend/windmill-api-integration-tests/tests/dbt_materialize_target.rs create mode 100644 backend/windmill-common/tests/dbt_producer_rules.rs diff --git a/backend/.sqlx/query-1d8effff5dd1e4a177efed7366e84411c125cbfada861992e3d6032de635bad6.json b/backend/.sqlx/query-1d8effff5dd1e4a177efed7366e84411c125cbfada861992e3d6032de635bad6.json new file mode 100644 index 0000000000..b60dd43fdf --- /dev/null +++ b/backend/.sqlx/query-1d8effff5dd1e4a177efed7366e84411c125cbfada861992e3d6032de635bad6.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings\n SET dbt_warehouses = '{\"main\": {\"resource_path\": \"u/test-user/wh\"}}'::jsonb\n WHERE workspace_id = 'test-workspace'", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "1d8effff5dd1e4a177efed7366e84411c125cbfada861992e3d6032de635bad6" +} diff --git a/backend/.sqlx/query-1f7608defb5748db687145750520cb1059f63c1ea51db3471c213018b2983704.json b/backend/.sqlx/query-1f7608defb5748db687145750520cb1059f63c1ea51db3471c213018b2983704.json new file mode 100644 index 0000000000..77b00d9daa --- /dev/null +++ b/backend/.sqlx/query-1f7608defb5748db687145750520cb1059f63c1ea51db3471c213018b2983704.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind)\n VALUES ('test-workspace', 'main/analytics/marts', 'dbt', 'w', 'u/test-user/project',\n 'script')", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "1f7608defb5748db687145750520cb1059f63c1ea51db3471c213018b2983704" +} diff --git a/backend/.sqlx/query-33aa15957f56281947963cd696f7f518433741305d891d99059184eaf39fa2db.json b/backend/.sqlx/query-33aa15957f56281947963cd696f7f518433741305d891d99059184eaf39fa2db.json new file mode 100644 index 0000000000..fbfab14017 --- /dev/null +++ b/backend/.sqlx/query-33aa15957f56281947963cd696f7f518433741305d891d99059184eaf39fa2db.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path FROM asset WHERE workspace_id = 'test-workspace' AND kind = 'dbt' AND usage_path = 'u/test-user/ingest' AND usage_access_type = 'w'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false + ] + }, + "hash": "33aa15957f56281947963cd696f7f518433741305d891d99059184eaf39fa2db" +} diff --git a/backend/.sqlx/query-49d905cae6ba42a3df62bad9385e613381cb4dcf7e851197150a3771ab43c99e.json b/backend/.sqlx/query-49d905cae6ba42a3df62bad9385e613381cb4dcf7e851197150a3771ab43c99e.json new file mode 100644 index 0000000000..24e563d9ad --- /dev/null +++ b/backend/.sqlx/query-49d905cae6ba42a3df62bad9385e613381cb4dcf7e851197150a3771ab43c99e.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH producer AS (\n SELECT 'dbt://' || a.path AS trigger_ref, a.usage_path, s.language\n FROM asset a\n JOIN script s ON s.workspace_id = a.workspace_id AND s.path = a.usage_path\n AND s.archived = false AND s.deleted = false\n WHERE a.workspace_id = $1 AND a.kind = 'dbt'\n AND a.usage_kind = 'script' AND a.usage_access_type IN ('w', 'rw')\n AND 'dbt://' || a.path = ANY($2)\n )\n SELECT DISTINCT st.trigger_ref || ' → ' || st.runnable_path AS \"edge!\"\n FROM script_trigger st\n WHERE st.workspace_id = $1 AND st.trigger_kind = 'asset'\n AND st.trigger_ref = ANY($2)\n AND EXISTS (SELECT 1 FROM producer p\n WHERE p.trigger_ref = st.trigger_ref\n AND p.usage_path <> st.runnable_path\n AND p.language = 'dbt')\n AND NOT EXISTS (SELECT 1 FROM producer p\n WHERE p.trigger_ref = st.trigger_ref\n AND p.usage_path <> st.runnable_path\n AND p.language <> 'dbt')\n ORDER BY 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "edge!", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "TextArray" + ] + }, + "nullable": [ + null + ] + }, + "hash": "49d905cae6ba42a3df62bad9385e613381cb4dcf7e851197150a3771ab43c99e" +} diff --git a/backend/.sqlx/query-6890e2be43ff8c653ce62f8cde1b9877190091923a2615d7d94bdf89a12e43c7.json b/backend/.sqlx/query-6890e2be43ff8c653ce62f8cde1b9877190091923a2615d7d94bdf89a12e43c7.json new file mode 100644 index 0000000000..07c923620a --- /dev/null +++ b/backend/.sqlx/query-6890e2be43ff8c653ce62f8cde1b9877190091923a2615d7d94bdf89a12e43c7.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO script (workspace_id, hash, path, summary, description, content, created_by,\n language)\n VALUES ('test-workspace', 1, 'u/test-user/project', '', '', '', 'test-user', 'dbt')", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "6890e2be43ff8c653ce62f8cde1b9877190091923a2615d7d94bdf89a12e43c7" +} diff --git a/backend/.sqlx/query-ae4c0e8aeeef90d08a9b9c919b881b0a03e44beea80da732a4ad7166cf4c06cf.json b/backend/.sqlx/query-ae4c0e8aeeef90d08a9b9c919b881b0a03e44beea80da732a4ad7166cf4c06cf.json new file mode 100644 index 0000000000..2e0e072867 --- /dev/null +++ b/backend/.sqlx/query-ae4c0e8aeeef90d08a9b9c919b881b0a03e44beea80da732a4ad7166cf4c06cf.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT trigger_ref FROM script_trigger WHERE workspace_id = 'test-workspace' AND runnable_path = 'u/test-user/consumer' AND trigger_kind = 'asset'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "trigger_ref", + "type_info": "Text" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false + ] + }, + "hash": "ae4c0e8aeeef90d08a9b9c919b881b0a03e44beea80da732a4ad7166cf4c06cf" +} diff --git a/backend/.sqlx/query-b8161f6481460bed1c985bedc7c638a42f03599ce5eb872cf601f1da2e76b946.json b/backend/.sqlx/query-b8161f6481460bed1c985bedc7c638a42f03599ce5eb872cf601f1da2e76b946.json new file mode 100644 index 0000000000..6b2ec01b23 --- /dev/null +++ b/backend/.sqlx/query-b8161f6481460bed1c985bedc7c638a42f03599ce5eb872cf601f1da2e76b946.json @@ -0,0 +1,63 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT s.path AS \"path!\", s.language AS \"language!: ScriptLang\"\n FROM asset a\n JOIN script s ON s.workspace_id = a.workspace_id AND s.path = a.usage_path\n AND s.archived = false AND s.deleted = false\n WHERE a.workspace_id = $1 AND a.kind = 'dbt' AND a.path = $2\n AND a.usage_kind = 'script' AND a.usage_access_type IN ('w', 'rw')\n AND a.usage_path <> ALL($3)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "language!: ScriptLang", + "type_info": { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java", + "duckdb", + "ruby", + "rlang", + "dbt" + ] + } + } + } + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "TextArray" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "b8161f6481460bed1c985bedc7c638a42f03599ce5eb872cf601f1da2e76b946" +} diff --git a/backend/.sqlx/query-e37b74f77cfa8769aee4d95155a3b3b6856b44d49c0712f08e307f950f210570.json b/backend/.sqlx/query-e37b74f77cfa8769aee4d95155a3b3b6856b44d49c0712f08e307f950f210570.json new file mode 100644 index 0000000000..46a9c8e505 --- /dev/null +++ b/backend/.sqlx/query-e37b74f77cfa8769aee4d95155a3b3b6856b44d49c0712f08e307f950f210570.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind)\n VALUES ('test-workspace', 'main/analytics/orders', 'dbt', 'w', 'u/test-user/project',\n 'script')", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "e37b74f77cfa8769aee4d95155a3b3b6856b44d49c0712f08e307f950f210570" +} diff --git a/backend/parsers/windmill-parser/src/asset_parser.rs b/backend/parsers/windmill-parser/src/asset_parser.rs index 843a83a310..30c6047255 100644 --- a/backend/parsers/windmill-parser/src/asset_parser.rs +++ b/backend/parsers/windmill-parser/src/asset_parser.rs @@ -29,10 +29,10 @@ pub enum AssetKind { Ducklake, DataTable, Volume, - /// A warehouse relation a dbt project builds or reads, - /// `dbt:////`, the warehouse named as the - /// workspace configures it. The scheme names the producer, the path stays - /// the relation — see `windmill_types::AssetKind::Dbt`. + /// A warehouse relation, `dbt:////`, the warehouse + /// named as the workspace configures it. The scheme names the namespace dbt + /// made — a script in any language but dbt's own can declare a write to one — + /// and the path stays the relation. See `windmill_types::AssetKind::Dbt`. Dbt, } @@ -288,11 +288,13 @@ pub struct RetrySpec { } // `// materialize [manual] [append] [key=] [history] [track=]` -// — declares that this script produces a *managed* materialization of `` -// (a `ducklake://` table). By default the runtime generates the write DDL around +// — declares that this script produces ``. A `ducklake://` table is +// materialized *managed* by default: the runtime generates the write DDL around // the script's single trailing `SELECT` and owns idempotency, partition-state // and snapshot capture. `manual` is the escape hatch: the script writes its own -// DDL and the runtime only records state (track-only). The reconciliation +// DDL and the runtime only records state (track-only) — and it is the only mode a +// `dbt://` warehouse relation has, since nothing generates warehouse DDL (deploy +// enforces that; see docs/dbt-runtime.md). The reconciliation // strategy options apply to managed mode: none → DELETE-by-partition + INSERT // (replace); `key=` → MERGE (dedup within slice, SCD type 1); `append` → // INSERT-only. `append` wins if both are given (deploy-time warning). @@ -804,6 +806,18 @@ pub fn canonicalize_table_asset_path(path: &str) -> String { ) } +/// Whether a `dbt://` path names a whole relation, `//`. +/// +/// Every producer spells one that way — the manifest ingest derives it from +/// `relation_name`, a `// materialize` target is checked against it — so anything +/// else can be produced by nothing and read by nothing. Both sides of the deploy +/// ask here rather than counting segments themselves: a subscription and a write +/// that disagreed on the shape would refuse and accept the same string. +pub fn is_full_relation_path(path: &str) -> bool { + let mut segments = path.split('/'); + segments.clone().count() == 3 && !segments.any(str::is_empty) +} + /// A doubled delimiter inside a quoted identifier is that delimiter, literally — /// the same rule the worker's `split_relation` applies to `relation_name`. Both /// have to decode it or one spelling of a table becomes two graph nodes: the dbt @@ -1740,10 +1754,7 @@ mod pipeline_annotation_tests { // just stop being the same node and the cross-boundary cascade never fires. #[test] fn table_paths_from_every_spelling_canonicalize_to_one_key() { - let canonical = Some(( - AssetKind::Dbt, - Cow::Owned("main/analytics/orders".into()), - )); + let canonical = Some((AssetKind::Dbt, Cow::Owned("main/analytics/orders".into()))); for spelling in [ // Hand-written annotation. "dbt://main/analytics/orders", @@ -1765,6 +1776,17 @@ mod pipeline_annotation_tests { } } + /// The shape both halves of the deploy check against: a subscription and a + /// write that disagreed on it would refuse and accept the same string. + #[test] + fn a_whole_relation_is_three_non_empty_segments() { + assert!(is_full_relation_path("main/analytics/orders")); + assert!(is_full_relation_path("main/archive.sales/orders")); + for partial in ["main", "main/analytics", "main/analytics/orders/x", "", "main//orders"] { + assert!(!is_full_relation_path(partial), "{partial} is not a relation"); + } + } + // A relation that overrode its database carries `.` in // one segment, and each half can be quoted independently. Stripping only // the outer pair leaves a key the manifest ingest never produces, so the @@ -1791,10 +1813,7 @@ mod pipeline_annotation_tests { // database qualifier. assert_eq!( parse_asset_syntax("dbt://main/\"sales.v2\"/orders", false), - Some(( - AssetKind::Dbt, - Cow::Owned("main/sales.v2/orders".into()) - )) + Some((AssetKind::Dbt, Cow::Owned("main/sales.v2/orders".into()))) ); } @@ -1813,14 +1832,8 @@ mod pipeline_annotation_tests { "dbt://main/analytics/\"order\"\"s\"", "main/analytics/order\"s", ), - ( - "dbt://main/`da``ta`/`orders`", - "main/da`ta/orders", - ), - ( - "dbt://main/[my]]schema]/[orders]", - "main/my]schema/orders", - ), + ("dbt://main/`da``ta`/`orders`", "main/da`ta/orders"), + ("dbt://main/[my]]schema]/[orders]", "main/my]schema/orders"), // And in one half of a database-qualified segment. ( "dbt://main/\"arch\"\"ive\".\"sales\"/orders", @@ -1839,8 +1852,14 @@ mod pipeline_annotation_tests { // apart. A lone delimiter treated as opening a quote would be dropped — // `sa"les` filed as `sales` — and the two derivations would split. for (decoded, spelled) in [ - ("dbt://main/sa\"les/orders", "dbt://main/\"sa\"\"les\"/orders"), - ("dbt://main/analytics/order\"s", "dbt://main/analytics/\"order\"\"s\""), + ( + "dbt://main/sa\"les/orders", + "dbt://main/\"sa\"\"les\"/orders", + ), + ( + "dbt://main/analytics/order\"s", + "dbt://main/analytics/\"order\"\"s\"", + ), ( "dbt://main/arch\"ive.sales/orders", "dbt://main/\"arch\"\"ive\".\"sales\"/orders", diff --git a/backend/windmill-api-integration-tests/tests/dbt_materialize_target.rs b/backend/windmill-api-integration-tests/tests/dbt_materialize_target.rs new file mode 100644 index 0000000000..60f846ca1f --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/dbt_materialize_target.rs @@ -0,0 +1,250 @@ +use serde_json::json; +use sqlx::{Pool, Postgres}; + +use windmill_test_utils::*; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder { + builder.header("Authorization", "Bearer SECRET_TOKEN") +} + +async fn deploy(port: u16, path: &str, content: &str) -> reqwest::Response { + authed(client().post(format!( + "http://localhost:{port}/api/w/test-workspace/scripts/create" + ))) + .json(&json!({ + "path": path, + "summary": "", + "description": "", + "content": content, + "language": "deno", + "schema": { "type": "object", "properties": {}, "required": [] } + })) + .send() + .await + .unwrap() +} + +/// A `dbt://` relation is one graph node only while every side spells it the same +/// way, and three sides derive that spelling independently: the `// materialize` +/// target becomes an `asset.path`, a `// on` ref becomes a `script_trigger`, and +/// the deploy-time refusal joins the two. The unit tests on `sole_dbt_producer` +/// prove the predicate; only a deploy proves the handler feeds it the key the +/// table actually holds — so a canonicalization that drifted on one side would +/// pass those and split the node here. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_dbt_materialize_target_deploy_contract(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + sqlx::query!( + r#"UPDATE workspace_settings + SET dbt_warehouses = '{"main": {"resource_path": "u/test-user/wh"}}'::jsonb + WHERE workspace_id = 'test-workspace'"# + ) + .execute(&db) + .await?; + + // Nothing generates warehouse DDL, so a managed target is refused rather than + // degraded into the track-only mode it would silently become. + let resp = deploy( + port, + "u/test-user/managed", + "// materialize dbt://main/analytics/orders\nexport async function main() {}", + ) + .await; + assert_eq!(resp.status(), 400); + assert!(resp.text().await?.contains("must be `manual`")); + + // The warehouse segment is the identity a dbt model keys on; a name the + // workspace does not configure strands the write on an unreachable node. + let resp = deploy( + port, + "u/test-user/unknown_wh", + "// materialize manual dbt://nope/analytics/orders\nexport async function main() {}", + ) + .await; + assert_eq!(resp.status(), 400); + assert!(resp.text().await?.contains("does not configure")); + + // Only the DuckDB executor runs `// data_test` probes, and it runs them around + // a managed write — so a declarer in another language would deploy green with + // its assertions silently never executed. + let resp = deploy( + port, + "u/test-user/tested", + "// materialize manual dbt://main/analytics/orders\n// data_test not_null id\nexport async function main() {}", + ) + .await; + assert_eq!(resp.status(), 400); + assert!(resp + .text() + .await? + .contains("`// data_test` is not supported")); + + // Both halves are held to the same relation: every producer is a whole + // `//` under a configured warehouse, so a + // subscription to anything else names something nothing can ever write. + for (path, ref_, expected) in [ + ( + "u/test-user/partial_sub", + "dbt://main/analytics", + "not a whole warehouse relation", + ), + ( + "u/test-user/unknown_wh_sub", + "dbt://nope/analytics/orders", + "does not configure", + ), + ] { + let resp = deploy( + port, + path, + &format!("// on {ref_}\nexport async function main() {{}}"), + ) + .await; + assert_eq!(resp.status(), 400); + assert!(resp.text().await?.contains(expected)); + } + + // Past `asset.path`'s column, where the manifest ingest drops the relation and + // no producer row can exist on either side — computed from the bound so it + // cannot drift under it. + let overlong = format!( + "main/analytics/{}", + "o".repeat(windmill_common::dbt_manifest::MAX_ASSET_PATH_LEN) + ); + let resp = deploy( + port, + "u/test-user/overlong_sub", + &format!("// on dbt://{overlong}\nexport async function main() {{}}"), + ) + .await; + assert_eq!(resp.status(), 400); + assert!(resp.text().await?.contains("characters an asset path holds")); + + // Any language may declare the write — the DuckLake write engine is DuckDB's, + // this declaration is not — and the target is canonicalized on the way into + // `asset`, so a hand-written mixed-case spelling lands on the model's key. + let resp = deploy( + port, + "u/test-user/ingest", + "// materialize manual dbt://main/ANALYTICS/Orders\nexport async function main() {}", + ) + .await; + assert_eq!(resp.status(), 201); + // The create response, not `{:x}` over the stored i64: `ScriptHash` decodes + // hex and demands 8 bytes, while `LowerHex` drops leading zeros, so a hash + // under 2^60 would 422 the rename below instead of reaching the refusal. + let ingest_hash = resp.text().await?; + let write = sqlx::query_scalar!( + "SELECT path FROM asset WHERE workspace_id = 'test-workspace' AND kind = 'dbt' \ + AND usage_path = 'u/test-user/ingest' AND usage_access_type = 'w'" + ) + .fetch_one(&db) + .await?; + assert_eq!(write, "main/analytics/orders"); + + // That producer is native, so subscribing to what it writes is accepted — and + // the `// on` ref has to canonicalize identically, or the row it stores names + // a relation nothing produces. + let resp = deploy( + port, + "u/test-user/consumer", + "// on dbt://main/\"Analytics\"/\"Orders\"\nexport async function main() {}", + ) + .await; + assert_eq!(resp.status(), 201); + let trigger_ref = sqlx::query_scalar!( + "SELECT trigger_ref FROM script_trigger WHERE workspace_id = 'test-workspace' \ + AND runnable_path = 'u/test-user/consumer' AND trigger_kind = 'asset'" + ) + .fetch_one(&db) + .await?; + assert_eq!(trigger_ref, "dbt://main/analytics/orders"); + + // With dbt as the only producer the same subscription can never be woken — a + // dbt run does not dispatch — so the deploy refuses it and names the project. + sqlx::query!( + "INSERT INTO script (workspace_id, hash, path, summary, description, content, created_by, + language) + VALUES ('test-workspace', 1, 'u/test-user/project', '', '', '', 'test-user', 'dbt')" + ) + .execute(&db) + .await?; + sqlx::query!( + "INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind) + VALUES ('test-workspace', 'main/analytics/marts', 'dbt', 'w', 'u/test-user/project', + 'script')" + ) + .execute(&db) + .await?; + let resp = deploy( + port, + "u/test-user/mart_consumer", + "// on dbt://main/analytics/MARTS\nexport async function main() {}", + ) + .await; + assert_eq!(resp.status(), 400); + assert!(resp.text().await?.contains("u/test-user/project")); + + // A rename is the other half of that: the producer's write still sits at the + // OLD path in the committed snapshot this deploy reads, while the same + // transaction removes it — so it must not count as the producer that would + // wake the subscription the rename adds. + sqlx::query!( + "INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind) + VALUES ('test-workspace', 'main/analytics/orders', 'dbt', 'w', 'u/test-user/project', + 'script')" + ) + .execute(&db) + .await?; + let resp = authed(client().post(format!( + "http://localhost:{port}/api/w/test-workspace/scripts/create" + ))) + .json(&json!({ + "path": "u/test-user/ingest_renamed", + "parent_hash": ingest_hash, + "summary": "", + "description": "", + "content": "// on dbt://main/analytics/orders\nexport async function main() {}", + "language": "deno", + "schema": { "type": "object", "properties": {}, "required": [] } + })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 400); + assert!(resp.text().await?.contains("u/test-user/project")); + + // Neither annotation is accepted on a dbt script: the graph ingest + // republishes that path's asset and trigger rows wholesale, so either would + // deploy something the dependency job then silently removes. + for content in [ + "# materialize manual dbt://main/analytics/orders\nprofile:\n warehouse: main\n", + "# on dbt://main/analytics/orders\nprofile:\n warehouse: main\n", + ] { + let resp = authed(client().post(format!( + "http://localhost:{port}/api/w/test-workspace/scripts/create" + ))) + .json(&json!({ + "path": "u/test-user/dbt_project", + "summary": "", + "description": "", + "content": content, + "language": "dbt", + "modules": { "dbt_project.yml": { "content": "name: p\n", "language": "dbt" } }, + "schema": { "type": "object", "properties": {}, "required": [] } + })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 400); + assert!(resp.text().await?.contains("a dbt script cannot")); + } + + Ok(()) +} diff --git a/backend/windmill-api-scripts/src/scripts.rs b/backend/windmill-api-scripts/src/scripts.rs index ccc5815aa7..d24b3c3a5c 100644 --- a/backend/windmill-api-scripts/src/scripts.rs +++ b/backend/windmill-api-scripts/src/scripts.rs @@ -39,8 +39,8 @@ use sqlx::{FromRow, Postgres, Transaction}; use std::{collections::HashMap, sync::Arc}; use windmill_audit::audit_oss::{audit_log, AuditAuthorable}; use windmill_audit::ActionKind; -use windmill_dep_map::{lock_hash::record_lock_hashes, process_relative_imports}; use windmill_dep_map::scoped_dependency_map::ScopedDependencyMap; +use windmill_dep_map::{lock_hash::record_lock_hashes, process_relative_imports}; use windmill_common::{ assets::{ @@ -1081,6 +1081,57 @@ fn lock_hash_entry(path: &str, lock: &str) -> [(String, i64); 1] { [(path.to_string(), hash_script(lock))] } +/// The `dbt://` relation both halves of a deploy have to agree on: a whole +/// `//`, under a warehouse this workspace configures. +/// +/// Every producer is held to exactly this — a `// materialize` target here, a +/// descriptor's `profile.warehouse` in the worker — so a subscription to anything +/// else names a relation nothing can ever write. No later deploy fixes that and +/// no dormant-edge warning reports it, since the warning fires on a dbt project's +/// ingest and no project can claim a relation under a warehouse that isn't there. +/// Asking here rather than at each site is what keeps the two from drifting into +/// refusing and accepting the same string. +async fn validate_dbt_relation( + db: &sqlx::Pool, + w_id: &str, + relation: &str, + what: &str, +) -> Result<()> { + if !windmill_parser::asset_parser::is_full_relation_path(relation) { + return Err(Error::BadRequest(format!( + "{what} `dbt://{relation}` is not a whole warehouse relation \ + (`dbt:////`)." + ))); + } + // `asset.path` is VARCHAR(255) and the manifest ingest drops a relation that + // outgrows it rather than failing the whole graph, so past the column no + // producer row can exist on either side — a write would be rejected by + // Postgres mid-deploy, and `script_trigger.trigger_ref` is unbounded text + // that would take the subscription and keep it dormant for good. + let max = windmill_common::dbt_manifest::MAX_ASSET_PATH_LEN; + if relation.chars().count() > max { + return Err(Error::BadRequest(format!( + "{what} `dbt://{relation}` is longer than the {max} characters an asset path \ + holds, so it cannot be recorded." + ))); + } + let warehouse = relation.split('/').next().unwrap_or_default(); + // Only the resolver's own "no such warehouse" is the annotation's fault. Its + // other failures — the query, and a setting entry with no `resource_path` — + // keep their own error: blaming the warehouse name for those misdescribes + // them, and flattening the malformed-setting one to a 400 hides a server + // fault behind a client one. + windmill_common::workspaces::dbt_warehouse_exists(db, w_id, warehouse) + .await + .map_err(|e| match e { + Error::NotFound(_) => Error::BadRequest(format!( + "{what} `dbt://{relation}` names a warehouse this workspace does not \ + configure: {e}" + )), + other => other, + }) +} + async fn create_script_internal<'c>( mut ns: NewScript, w_id: String, @@ -1565,34 +1616,90 @@ async fn create_script_internal<'c>( // membership; parsed writes tell us what is produced (we don't record // them in auto_kind itself). let pipeline_annotations = parse_pipeline_annotations(&ns.content); - // `// materialize` materializes a `ducklake:///` target from a - // DuckDB script. These two constraints hold for *both* modes: a non-DuckLake - // target would otherwise deploy, register a producer in the asset graph, then - // silently no-op at run time (`build_materialized_query` returns `Ok(None)`), - // and a non-DuckDB script never reaches the executor that records state. The - // managed-only checks (single trailing SELECT, no SQL args) come after — a - // `manual` script owns its DDL and skips them. + // `// materialize` names what this script produces. Two target kinds, and the + // runtime behind each is what constrains the annotation: + // • `ducklake:///
` — the DuckDB executor generates the write + // (or, in `manual` mode, records the state the script wrote itself), so + // the script has to be a DuckDB one and the target has to name a table. + // A non-DuckDB script never reaches that executor. + // • `dbt:////` — a warehouse relation. Nothing + // generates warehouse DDL, so the declaration is track-only (`manual`) + // and any language but dbt's own may make it: the script writes the + // relation, the worker records the materialization, and the relation's + // asset node is shared with whatever dbt model reads it. A dbt project's + // own writes are read from its manifest, so it may not declare one. + // Any other kind would deploy, register a producer in the asset graph, then + // silently no-op at run time (`build_materialized_query` returns `Ok(None)`). + // The managed-only checks (single trailing SELECT, no SQL args) come after — + // a `manual` script owns its DDL and skips them. if let Some(m) = pipeline_annotations.materialize.as_ref() { - if ns.language != ScriptLang::DuckDb { - return Err(Error::BadRequest(format!( - "`// materialize` is only supported for DuckDB scripts, not {}. Use the \ - wmll.ducklake helpers to materialize from other languages.", - ns.language.as_str() - ))); - } - if m.target_kind != windmill_parser::asset_parser::AssetKind::Ducklake { + use windmill_parser::asset_parser::AssetKind as PAssetKind; + // The producer half of the rule the trigger loop below applies to `// on`: + // a dbt project's writes come from its manifest, and the graph ingest + // republishes this path's asset rows wholesale, so a declared one would be + // wiped by the very deploy that accepted it while its runs kept stamping + // the relation. + if ns.language == ScriptLang::Dbt { return Err(Error::BadRequest( - "`// materialize` only supports a DuckLake target \ - (`ducklake:///
`); other asset kinds aren't materializable." + "a dbt script cannot declare `// materialize`: what a project builds is read \ + from its manifest and published by the graph ingest, not annotated." .to_string(), )); } - if !m.target_path.contains('/') { - return Err(Error::BadRequest(format!( - "`// materialize` needs a table in the target: \ - `ducklake://{0}/
` (got `ducklake://{0}`).", - m.target_path - ))); + match m.target_kind { + PAssetKind::Ducklake => { + if ns.language != ScriptLang::DuckDb { + return Err(Error::BadRequest(format!( + "`// materialize` is only supported for DuckDB scripts, not {}. Use the \ + wmll.ducklake helpers to materialize from other languages, or declare a \ + warehouse relation with `// materialize manual dbt://…`.", + ns.language.as_str() + ))); + } + if !m.target_path.contains('/') { + return Err(Error::BadRequest(format!( + "`// materialize` needs a table in the target: \ + `ducklake://{0}/
` (got `ducklake://{0}`).", + m.target_path + ))); + } + } + PAssetKind::Dbt => { + // `// data_test` runs as verifier probes the DuckDB executor + // splices around a MANAGED write. Nothing generates a warehouse + // write, so nothing would run them — and unlike the DuckLake + // `manual` case, which at least fails loudly in that executor, a + // declarer in another language would deploy green with its + // data-quality assertions silently never executed. + if !pipeline_annotations.data_tests.is_empty() { + return Err(Error::BadRequest( + "`// data_test` is not supported with a `dbt://` target: the checks run \ + against a managed materialization, and a warehouse relation is \ + written by the script itself. Assert on the relation with a dbt \ + test in the project that reads it." + .to_string(), + )); + } + if !m.manual { + return Err(Error::BadRequest( + "`// materialize dbt://…` must be `manual`: Windmill generates no \ + warehouse DDL, so the script issues its own write and only the outcome \ + is recorded. Write \ + `// materialize manual dbt:////`." + .to_string(), + )); + } + validate_dbt_relation(&db, &w_id, &m.target_path, "`// materialize` target") + .await?; + } + _ => { + return Err(Error::BadRequest( + "`// materialize` only supports a DuckLake (`ducklake:///
`) or \ + warehouse-relation (`dbt:////`) target; other asset \ + kinds aren't materializable." + .to_string(), + )); + } } if !m.manual { if let Err(e) = windmill_parser::sql_materialize::classify_wrap(&ns.content) { @@ -2375,16 +2482,43 @@ async fn create_script_internal<'c>( let Some((trigger_kind, trigger_ref)) = trigger_spec_to_row(spec) else { continue; }; - // A `dbt://` subscription can never fire: dbt is the only producer of a - // warehouse relation (`// materialize` takes DuckLake targets only) and a - // dbt run does not dispatch. Refusing beats persisting a row that draws a - // cascade arrow on the canvas and then never wakes anything. - if trigger_ref.starts_with("dbt://") { - return Err(Error::BadRequest(format!( - "`{trigger_ref}` cannot be subscribed to: a dbt run does not trigger downstream \ - runs, and nothing else writes a warehouse relation. Declare the read without \ - `on` to keep the lineage edge, or schedule this script." - ))); + // A `dbt://` subscription fires only when a NON-dbt job materialized the + // relation: `// materialize manual dbt://…` declares such a write, while a + // dbt run records its models and does not dispatch. So refuse exactly the + // edge that cannot fire — one whose relation is already claimed by dbt and + // by nothing else — rather than every `dbt://` edge (`sole_dbt_producer`, + // which takes the workspace pool: under RLS an unreadable native producer + // would refuse a live subscription). + if let Some(relation) = trigger_ref.strip_prefix("dbt://") { + // The subscriber side of the same rule: a dbt project is not woken by + // the asset cascade. Its graph ingest clears these rows for its own + // path, so accepting one here would deploy an edge the dependency job + // then silently removes. + if ns.language == ScriptLang::Dbt { + return Err(Error::BadRequest(format!( + "a dbt script cannot subscribe to `{trigger_ref}`: dbt orders its own DAG \ + and a project is run on its schedule, not woken by an asset cascade." + ))); + } + validate_dbt_relation(&db, &w_id, relation, "subscription target").await?; + // Both paths under a rename: the old one's committed write row is + // still there and this transaction is about to remove it. + let deploying_paths = match p_path_opt.as_deref().filter(|old| *old != ns.path) { + Some(old) => vec![ns.path.clone(), old.to_string()], + None => vec![ns.path.clone()], + }; + if let Some(dbt_owner) = + windmill_common::assets::sole_dbt_producer(&db, &w_id, relation, &deploying_paths) + .await? + { + return Err(Error::BadRequest(format!( + "`{trigger_ref}` cannot be subscribed to: it is built by the dbt project at \ + `{dbt_owner}`, and a dbt run does not trigger downstream runs. Declare the \ + read without `on` to keep the lineage edge, or schedule this script. A \ + relation written by a `// materialize manual {trigger_ref}` script can be \ + subscribed to." + ))); + } } // Effective debounce for this edge: per-`// on debounce=` wins, // else the script-level `// debounce` default. Debounce only diff --git a/backend/windmill-common/src/assets.rs b/backend/windmill-common/src/assets.rs index 117b41632a..4d5523524c 100644 --- a/backend/windmill-common/src/assets.rs +++ b/backend/windmill-common/src/assets.rs @@ -175,10 +175,12 @@ fn is_write_access(access: Option) -> bool { /// producers). Resource / datatable / volume reads stay explicit-`// on`: /// a config/lookup read cascading is more often surprising than wanted. fn is_auto_trigger_kind(kind: AssetKind) -> bool { - // `Dbt` is deliberately NOT here. dbt is the only thing that can produce a - // warehouse relation (`// materialize` takes DuckLake targets only) and a dbt - // run does not dispatch, so a derived `dbt://` edge could never fire — it - // would draw a cascade arrow into a script nothing can wake. + // `Dbt` is deliberately NOT here. A warehouse relation is usually built by + // the dbt project that reads it, and a dbt run does not dispatch, so deriving + // an edge from every `dbt://` read would draw cascade arrows that mostly never + // fire. The relations a native `// materialize manual dbt://…` script writes + // do wake subscribers, but only through an explicit `// on`, which is where + // the author states that this particular relation has such a producer. matches!(kind, AssetKind::Ducklake | AssetKind::S3Object) } @@ -235,6 +237,134 @@ pub fn derive_pipeline_asset_trigger_refs( out } +/// A dbt script that builds the `dbt://` relation at `asset_path`, when dbt is +/// its ONLY producer. +/// +/// That is the one shape in which subscribing to a warehouse relation can never +/// be woken: a dbt run records the models it built and does not dispatch +/// (`asset_dispatch` returns early for `ScriptLang::Dbt`), while a script that +/// declares `// materialize manual dbt://…` fans out on the ordinary path. +/// +/// `None` covers both "some non-dbt script materializes it" and "nothing +/// produces it yet" — the second is the ordinary deploy-order case, identical to +/// every other asset kind, not a dormant edge. +/// +/// **Give it the workspace pool, not an RLS-scoped transaction.** `script` +/// carries RLS while `asset` does not, so a scoped executor hides producers, and +/// the hidden ones fail in the harmful direction: a native producer the deployer +/// cannot read leaves a dbt-only set behind and refuses a subscription that would +/// have fired. What it discloses in exchange is the path of a dbt script building +/// a relation the caller already named, which the workspace asset graph hands out +/// for every `dbt://` node anyway (the source that script wrote stays gated). +/// Callers must therefore already be scoped to `workspace_id`. +/// +/// `deploying_paths` is excluded from the producer set, and has to be: reading +/// committed rows means the deploying script's own are the version being +/// replaced, so one that just dropped its `// materialize` would still count as a +/// producer and let a now-dormant subscription through. Pass every path this +/// deploy is rewriting — under a rename that is the old path as well as the new +/// one, whose committed write row the transaction is about to remove. Excluding +/// them is free of the opposite error, because a script never wakes its own +/// subscription — the dispatcher skips that as a self-loop. +/// +/// A producer another deploy is committing concurrently is invisible either way, +/// and the outcome depends on which side it is. An uncommitted NATIVE producer +/// leaves a dbt-only set and refuses, with a message the user can retry past. An +/// uncommitted DBT one leaves an empty set and accepts — and if that ingest then +/// commits and runs [`dormant_dbt_subscriptions`] before this deploy's trigger +/// row lands, neither side reports the edge it left dormant. Serializing the two +/// is not worth it: they would have to share a per-relation lock, and the ingest +/// takes `script … FOR UPDATE` before its own advisory lock, so a deploy holding +/// relation locks first inverts that order into a deadlock across the two +/// subsystems. The next deploy of that project warns (docs/dbt-runtime.md). +pub async fn sole_dbt_producer<'e>( + executor: impl PgExecutor<'e>, + workspace_id: &str, + asset_path: &str, + deploying_paths: &[String], +) -> error::Result> { + use crate::scripts::ScriptLang; + let producers = sqlx::query!( + r#"SELECT s.path AS "path!", s.language AS "language!: ScriptLang" + FROM asset a + JOIN script s ON s.workspace_id = a.workspace_id AND s.path = a.usage_path + AND s.archived = false AND s.deleted = false + WHERE a.workspace_id = $1 AND a.kind = 'dbt' AND a.path = $2 + AND a.usage_kind = 'script' AND a.usage_access_type IN ('w', 'rw') + AND a.usage_path <> ALL($3)"#, + workspace_id, + asset_path, + deploying_paths + ) + .fetch_all(executor) + .await?; + if producers + .iter() + .any(|p| !matches!(p.language, ScriptLang::Dbt)) + { + return Ok(None); + } + Ok(producers.into_iter().next().map(|p| p.path)) +} + +/// The set form of [`sole_dbt_producer`], for asking about many relations at +/// once: every `// on dbt://` edge among `relations` whose producers +/// are all dbt scripts, rendered as `dbt:// → `. +/// +/// A dbt deploy asks this about the relations it just ingested, because that +/// ingest is what can retroactively leave a subscription accepted earlier — when +/// nothing produced the relation — with dbt as its only producer. +/// +/// Spells the predicate the same way its singular sibling does, per subscriber: +/// the producer set excludes the subscriber's own path (a script never wakes +/// itself) and has to be non-empty (nothing produces it yet is deploy order, not +/// a dormant edge). Two "is dbt the sole producer" rules that drifted apart would +/// silence this warning with nothing failing. +/// +/// Same disclosure and executor contract as [`sole_dbt_producer`]: workspace +/// pool, caller already scoped to `workspace_id`. +pub async fn dormant_dbt_subscriptions<'e>( + executor: impl PgExecutor<'e>, + workspace_id: &str, + relations: &[String], +) -> error::Result> { + if relations.is_empty() { + return Ok(vec![]); + } + let refs = relations + .iter() + .map(|r| format!("dbt://{r}")) + .collect::>(); + Ok(sqlx::query_scalar!( + r#"WITH producer AS ( + SELECT 'dbt://' || a.path AS trigger_ref, a.usage_path, s.language + FROM asset a + JOIN script s ON s.workspace_id = a.workspace_id AND s.path = a.usage_path + AND s.archived = false AND s.deleted = false + WHERE a.workspace_id = $1 AND a.kind = 'dbt' + AND a.usage_kind = 'script' AND a.usage_access_type IN ('w', 'rw') + AND 'dbt://' || a.path = ANY($2) + ) + SELECT DISTINCT st.trigger_ref || ' → ' || st.runnable_path AS "edge!" + FROM script_trigger st + WHERE st.workspace_id = $1 AND st.trigger_kind = 'asset' + AND st.trigger_ref = ANY($2) + AND EXISTS (SELECT 1 FROM producer p + WHERE p.trigger_ref = st.trigger_ref + AND p.usage_path <> st.runnable_path + AND p.language = 'dbt') + AND NOT EXISTS (SELECT 1 FROM producer p + WHERE p.trigger_ref = st.trigger_ref + AND p.usage_path <> st.runnable_path + AND p.language <> 'dbt') + ORDER BY 1"#, + workspace_id, + &refs + ) + .fetch_all(executor) + .await?) +} + /// Clear and reinsert the full static-asset usage set of a script in one tx, /// invalidating the producer-writes cache at most once and only on a real /// change. The cache (asset_dispatch::ASSET_PRODUCER_WRITES_CACHE) keys a diff --git a/backend/windmill-common/src/dbt_manifest.rs b/backend/windmill-common/src/dbt_manifest.rs index 8fb124a33f..e85dfc1910 100644 --- a/backend/windmill-common/src/dbt_manifest.rs +++ b/backend/windmill-common/src/dbt_manifest.rs @@ -8,9 +8,11 @@ //! Two things this module is deliberate about: //! //! * **Asset identity is the physical relation.** A model becomes -//! `dbt:////`: the scheme names dbt, which is the -//! only thing that creates one, but the PATH is the relation and never dbt's -//! own `unique_id`. Two projects meet at a handoff — one materializes a mart, +//! `dbt:////`: the scheme names the namespace dbt +//! made — it is the only thing that DERIVES one, while any other language can +//! declare a write to one (decision 25) — but the PATH is the relation and +//! never dbt's own `unique_id`. Two projects meet at a handoff — one +//! materializes a mart, //! the next declares it a `source` — where `model.a.orders` and //! `source.b.analytics.orders` differ but the relation does not, so keying on //! the node id would leave each project an island; a native script reading the diff --git a/backend/windmill-common/tests/dbt_producer_rules.rs b/backend/windmill-common/tests/dbt_producer_rules.rs new file mode 100644 index 0000000000..f63e0879b5 --- /dev/null +++ b/backend/windmill-common/tests/dbt_producer_rules.rs @@ -0,0 +1,196 @@ +/*! + * One rule, two spellings: "is dbt the sole producer of this warehouse + * relation". `sole_dbt_producer` decides whether a `// on dbt://…` subscription + * is refused at deploy; `dormant_dbt_subscriptions` names the edges a dbt deploy + * retroactively leaves unwakeable. Both must answer "yes, dormant" only when + * every script writing the relation is a dbt one — a dbt run does not dispatch — + * and "no" both when a native `// materialize manual dbt://…` producer exists and + * when nothing produces the relation yet, which is the ordinary deploy-order + * case. Every way of getting this wrong is silent: a dormant edge on the canvas, + * a refused deploy of a valid pipeline, or a warning that stops appearing. + */ + +use sqlx::{Pool, Postgres}; +use windmill_common::assets::{dormant_dbt_subscriptions, sole_dbt_producer}; + +const WS: &str = "test-workspace"; +const RELATION: &str = "main/analytics/orders"; +const SUBSCRIBER: &str = "u/test-user/consumer"; + +async fn plant_producer(db: &Pool, path: &str, language: &str, hash: i64) { + sqlx::query( + "INSERT INTO script (workspace_id, hash, path, summary, description, content, created_by, + language) + VALUES ($1, $2, $3, '', '', '', 'test-user', $4::text::script_lang)", + ) + .bind(WS) + .bind(hash) + .bind(path) + .bind(language) + .execute(db) + .await + .expect("insert script"); + sqlx::query( + "INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind) + VALUES ($1, $2, 'dbt', 'w', $3, 'script') ON CONFLICT DO NOTHING", + ) + .bind(WS) + .bind(RELATION) + .bind(path) + .execute(db) + .await + .expect("insert asset"); +} + +async fn plant_subscriber(db: &Pool, path: &str) { + sqlx::query( + "INSERT INTO script_trigger (workspace_id, runnable_kind, runnable_path, trigger_kind, + trigger_ref) + VALUES ($1, 'script', $2, 'asset', 'dbt://' || $3)", + ) + .bind(WS) + .bind(path) + .bind(RELATION) + .execute(db) + .await + .expect("insert script_trigger"); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn no_producer_is_not_dormant(db: Pool) { + assert_eq!( + sole_dbt_producer(&db, WS, RELATION, &[SUBSCRIBER.to_string()]) + .await + .unwrap(), + None, + "a relation nothing produces yet must not refuse the subscription" + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn dbt_only_producer_is_dormant(db: Pool) { + plant_producer(&db, "u/test-user/project", "dbt", 1).await; + assert_eq!( + sole_dbt_producer(&db, WS, RELATION, &[SUBSCRIBER.to_string()]) + .await + .unwrap(), + Some("u/test-user/project".to_string()) + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn a_native_producer_beside_dbt_is_not_dormant(db: Pool) { + plant_producer(&db, "u/test-user/project", "dbt", 1).await; + plant_producer(&db, "u/test-user/ingest", "postgresql", 2).await; + assert_eq!( + sole_dbt_producer(&db, WS, RELATION, &[SUBSCRIBER.to_string()]) + .await + .unwrap(), + None + ); +} + +/// `asset` is keyed by path while `script` holds every version of it, so the +/// language has to be read off the live one: a path converted to dbt still has +/// its old native versions sitting in `script`. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn a_superseded_native_version_does_not_count(db: Pool) { + plant_producer(&db, "u/test-user/project", "postgresql", 1).await; + sqlx::query("UPDATE script SET archived = true WHERE hash = 1") + .execute(&db) + .await + .expect("archive the old version"); + plant_producer(&db, "u/test-user/project", "dbt", 2).await; + assert_eq!( + sole_dbt_producer(&db, WS, RELATION, &[SUBSCRIBER.to_string()]) + .await + .unwrap(), + Some("u/test-user/project".to_string()) + ); +} + +/// The rows of the script being deployed describe the version it replaces, so a +/// script dropping its `// materialize` while adding a subscription would +/// otherwise count itself as the producer that wakes it — and commit a dormant +/// edge. It can never be that producer anyway: the dispatcher skips self-loops. +/// Under a rename that write sits at the OLD path, which the deploy is removing +/// in the same uncommitted transaction, so both paths have to be excluded. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn the_subscriber_is_never_its_own_producer(db: Pool) { + plant_producer(&db, "u/test-user/project", "dbt", 1).await; + plant_producer(&db, SUBSCRIBER, "postgresql", 2).await; + assert_eq!( + sole_dbt_producer(&db, WS, RELATION, &[SUBSCRIBER.to_string()]) + .await + .unwrap(), + Some("u/test-user/project".to_string()) + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn a_renamed_producer_is_excluded_too(db: Pool) { + plant_producer(&db, "u/test-user/project", "dbt", 1).await; + plant_producer(&db, "u/test-user/old_ingest", "postgresql", 2).await; + assert_eq!( + sole_dbt_producer( + &db, + WS, + RELATION, + &[SUBSCRIBER.to_string(), "u/test-user/old_ingest".to_string()] + ) + .await + .unwrap(), + Some("u/test-user/project".to_string()), + "the write this deploy is moving off the old path cannot wake the subscription" + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn the_set_form_agrees_with_the_singular_one(db: Pool) { + let relations = vec![RELATION.to_string()]; + plant_subscriber(&db, "u/test-user/consumer").await; + plant_producer(&db, "u/test-user/project", "dbt", 1).await; + assert_eq!( + dormant_dbt_subscriptions(&db, WS, &relations) + .await + .unwrap(), + vec![format!("dbt://{RELATION} → u/test-user/consumer")], + "dbt alone builds it, so the subscription can never be woken" + ); + + plant_producer(&db, "u/test-user/ingest", "postgresql", 2).await; + assert!( + dormant_dbt_subscriptions(&db, WS, &relations) + .await + .unwrap() + .is_empty(), + "a native producer wakes it, so the edge is live" + ); +} + +/// The two ways the set form could stop meaning what the singular one means: a +/// relation nothing produces is deploy order rather than a dormant edge, and a +/// subscriber's own write is not a producer that can wake it — the dispatcher +/// skips self-loops, so that edge is dormant and has to be named. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn the_set_form_matches_on_the_edge_cases_too(db: Pool) { + let relations = vec![RELATION.to_string()]; + plant_subscriber(&db, SUBSCRIBER).await; + assert!( + dormant_dbt_subscriptions(&db, WS, &relations) + .await + .unwrap() + .is_empty(), + "nothing produces it yet, so nothing is dormant" + ); + + plant_producer(&db, "u/test-user/project", "dbt", 1).await; + plant_producer(&db, SUBSCRIBER, "postgresql", 2).await; + assert_eq!( + dormant_dbt_subscriptions(&db, WS, &relations) + .await + .unwrap(), + vec![format!("dbt://{RELATION} → {SUBSCRIBER}")], + "the subscriber's own write cannot wake it, so dbt is still the sole producer" + ); +} diff --git a/backend/windmill-queue/src/asset_dispatch.rs b/backend/windmill-queue/src/asset_dispatch.rs index af9ca957b9..a3f7154cda 100644 --- a/backend/windmill-queue/src/asset_dispatch.rs +++ b/backend/windmill-queue/src/asset_dispatch.rs @@ -248,11 +248,13 @@ async fn try_dispatch(db: &DB, job: &MiniCompletedJob) -> Result // A dbt run records the relations it builds, so it looks like a producer // here — but dbt does not trigger downstream runs. Its own DAG is dbt's to // order; the only thing a cascade would add is waking Windmill scripts that - // read a mart, and nothing outside dbt can declare a `dbt://` write, so - // that edge exists in one direction only. Cascading from a project whose - // per-run selection can build any subset of itself needs a per-run write set - // to be correct, which is a design worth doing deliberately rather than - // inferring. Until then dbt materializes and reports; it does not dispatch. + // read a mart. Cascading from a project whose per-run selection can build any + // subset of itself needs a per-run write set to be correct, which is a design + // worth doing deliberately rather than inferring: the deploy-time write set is + // not what ran, and the per-relation state table keeps one row per relation. + // Until then dbt materializes and reports; it does not dispatch. The opposite + // direction does: a native `// materialize manual dbt://…` script reaches the + // fan-out below on the ordinary path, on the strength of its own asset rows. if job.script_lang == Some(ScriptLang::Dbt) { return Ok(DispatchResult::default()); } diff --git a/backend/windmill-types/src/assets.rs b/backend/windmill-types/src/assets.rs index 463a365b06..166406b632 100644 --- a/backend/windmill-types/src/assets.rs +++ b/backend/windmill-types/src/assets.rs @@ -14,17 +14,20 @@ pub enum AssetKind { Ducklake, DataTable, Volume, - /// A warehouse relation a dbt project builds or reads, - /// `dbt:////`, where `` is the name the - /// workspace configures it under. + /// A warehouse relation, `dbt:////`, where + /// `` is the name the workspace configures it under. /// - /// The SCHEME names the producer — dbt is the only thing that creates one — - /// while the PATH stays the physical relation, because that is what two - /// projects agree on: a mart one builds is a `source` the next reads, and - /// their dbt `unique_id`s differ (`model.a.orders` vs - /// `source.b.analytics.orders`) where the relation does not - /// (docs/dbt-runtime.md, decision 11). A dbt run does not trigger that - /// reader — the shared node is lineage, not a cascade edge. + /// The SCHEME names the namespace dbt made rather than an exclusive + /// producer: dbt is what derives these relations from a project, and a script + /// in any language but dbt's own can DECLARE one it writes + /// (`// materialize manual dbt://…`) — a project's writes are read from its + /// manifest, never annotated. The PATH stays the physical relation, + /// because that is what two producers agree on: a mart one builds is a + /// `source` the next reads, and their dbt `unique_id`s differ + /// (`model.a.orders` vs `source.b.analytics.orders`) where the relation does + /// not (docs/dbt-runtime.md, decision 11). A dbt run does not trigger the + /// readers of what it built — that shared node is lineage, not a cascade + /// edge — while a declared write does (decision 25). Dbt, } diff --git a/backend/windmill-worker/src/dbt_executor.rs b/backend/windmill-worker/src/dbt_executor.rs index 33ddd6c32e..f302df6a1a 100644 --- a/backend/windmill-worker/src/dbt_executor.rs +++ b/backend/windmill-worker/src/dbt_executor.rs @@ -682,6 +682,7 @@ pub(crate) async fn dbt_dep( &conn, ) .await; + warn_dormant_subscribers(db, w_id, job_id, &ingested, &conn).await; } !published } else { @@ -3177,7 +3178,7 @@ async fn ingest_from_run( let snapshot_job = p.graph_refresh.snapshot_job(job.id); match conn { Connection::Sql(db) => { - persist_ingest( + let published = persist_ingest( db, &job.workspace_id, script_path, @@ -3190,6 +3191,13 @@ async fn ingest_from_run( p.graph_refresh.publishes_ownership(), ) .await?; + // Publishing ownership from a RUN makes this project the owner of + // those relations exactly as a deploy does, so it can be what leaves + // a subscription accepted while the relation had no producer with dbt + // as its only one. Same warning the deploy emits. + if published && p.graph_refresh.publishes_ownership() { + warn_dormant_subscribers(db, &job.workspace_id, &job.id, &ingested, conn).await; + } } // An agent worker reaches these tables only through the API. Publishing // is the whole of what it needs: a worker that can replace the graph @@ -3242,7 +3250,7 @@ enum GraphPublisher { /// Replace this script's graph, unless a newer version of it has been deployed. /// /// Write one ingest: the sidecar rows and the `asset` usages the manifest -/// implies. No subscriptions — a `dbt://` one could never fire. +/// implies. No subscriptions — a dbt project is not woken by the cascade. /// /// Returns whether this job was still the one entitled to the path-keyed half — /// false once a newer version has superseded it, or once the version is gone. @@ -3333,9 +3341,10 @@ async fn persist_ingest( &ingested.assets, ) .await?; - // A `dbt://` subscription can never fire, so none are derived from the - // manifest. The delete stays to clear what earlier versions wrote, which would - // otherwise keep drawing cascade arrows that wake nothing. + // A dbt project is not woken by the asset cascade (refused at deploy), so + // none are derived from the manifest either. The delete stays to clear what + // earlier versions wrote, which would otherwise keep drawing cascade arrows + // that wake nothing. sqlx::query!( "DELETE FROM script_trigger WHERE workspace_id = $1 AND runnable_kind = 'script' AND runnable_path = $2 @@ -3349,6 +3358,56 @@ async fn persist_ingest( Ok(true) } +/// Log the `// on dbt://…` subscriptions this project's relations leave dormant. +/// +/// Subscribing to a relation dbt already owns is refused at the subscriber's +/// deploy, but one deployed while nothing produced that relation is accepted — +/// as it is for every other asset kind — and an ingest is what can afterwards +/// make dbt its only producer. A dbt run does not dispatch, so such an edge is +/// drawn on the canvas and never fires; a job's own log is where that ordering is +/// visible. +/// +/// Called from both points that publish ownership, the deploy and a run whose +/// static descriptor found its profile moved — either can be the one that claims +/// the relation. +async fn warn_dormant_subscribers( + db: &sqlx::Pool, + w_id: &str, + job_id: &Uuid, + ingested: &windmill_common::dbt_manifest::IngestedManifest, + conn: &Connection, +) { + use windmill_common::assets::AssetUsageAccessType; + let relations: Vec = ingested + .assets + .iter() + .filter(|a| { + matches!( + a.access_type.or(a.alt_access_type), + Some(AssetUsageAccessType::W) | Some(AssetUsageAccessType::RW) + ) + }) + .map(|a| a.path.clone()) + .collect(); + match windmill_common::assets::dormant_dbt_subscriptions(db, w_id, &relations).await { + Ok(edges) if !edges.is_empty() => { + append_logs( + job_id, + w_id, + format!( + "\nThese subscriptions will not fire — a dbt run does not trigger downstream \ + runs, and nothing else writes their relation:\n {}\n", + edges.join("\n ") + ), + conn, + ) + .await; + } + Ok(_) => {} + Err(e) => tracing::warn!("listing dormant `dbt://` subscribers failed: {e:#}"), + } +} + /// Serialize publishers for one script path and confirm this job's version is /// still the newest. Both happen inside the caller's transaction, so a newer /// publisher either commits before this check sees it, or waits behind it and diff --git a/backend/windmill-worker/src/duckdb_executor.rs b/backend/windmill-worker/src/duckdb_executor.rs index 54457f81b0..b2e69d26d6 100644 --- a/backend/windmill-worker/src/duckdb_executor.rs +++ b/backend/windmill-worker/src/duckdb_executor.rs @@ -112,7 +112,9 @@ async fn fetch_custom_test_body(conn: &Connection, w_id: &str, path: &str) -> Re // the user's own ATTACH. `// data_test` lines append verifier probes that run // against the freshly-materialized target and raise (failing the run) on // violation. Returns `None` when there is no materialize annotation or the -// target isn't a ducklake (only ducklake is materialized in v1). +// target isn't a ducklake: only ducklake has a write engine, and a `dbt://` +// target is recorded by the generic job path instead (worker.rs, +// `record_declared_warehouse_write`). fn build_materialized_query( query: &str, partition_value: Option<&str>, diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 7e620439b0..8dc52ef936 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -5527,7 +5527,7 @@ async fn handle_code_execution_job( .await?; let language = language.clone(); - run_language_executor( + let result = run_language_executor( job, conn, client, @@ -5553,7 +5553,110 @@ async fn handle_code_execution_job( false, in_pipeline, ) - .await + .await; + record_declared_warehouse_write(job, conn, code, &result).await; + result +} + +/// Record the outcome of a `// materialize manual dbt:////` +/// declaration, the way the DuckDB executor records a DuckLake target. +/// +/// Nothing generates warehouse DDL, so the script issues its own write and this +/// is the only thing that turns it into a `materialized_partition` row — the +/// relation's last writer on the run page and the graph. Language-agnostic on +/// purpose — the DuckLake write engine is DuckDB's, this declaration is anyone's +/// — except dbt's own, which is refused at deploy. +/// +/// Best-effort, and it can be: the cascade fans out from the deploy-time `asset` +/// rows, not from this one, so a lost row costs the relation its last writer and +/// nothing else. It must not fail a job whose write already landed. +/// +/// Shares the reach of every other runtime pipeline annotation, which is this +/// function's caller: a job handed to a dedicated worker or a flow runner never +/// passes through it, so — exactly as `// partitioned` is not resolved there — +/// such a run performs its write and records no row. +async fn record_declared_warehouse_write( + job: &MiniPulledJob, + conn: &Connection, + code: &str, + result: &error::Result>, +) { + use windmill_common::materialization::{ + MaterializationStatus, RecordMaterializationRequest, UNPARTITIONED, + }; + // A DEPLOYED script only. The annotation is a deploy-time contract — `manual`, + // a three-segment relation, a configured warehouse — checked in + // `create_script_internal`, which also required write access to the path. A + // preview, hub or inline-flow body reaches this function without any of that, + // so honouring it there would let `jobs:run` alone restamp any relation's last + // writer from a script that never touched it. + if job.kind != JobKind::Script { + return; + } + // Cheap guard: the annotation scan is skipped for the overwhelming majority + // of jobs, which carry no `materialize` line at all. + if !code.contains("materialize") { + return; + } + let Some(m) = windmill_parser::asset_parser::parse_pipeline_annotations(code) + .materialize + .filter(|m| m.target_kind == windmill_parser::asset_parser::AssetKind::Dbt) + else { + return; + }; + // The slice this run wrote, resolved once upstream (`resolve_partition_for_job`) + // and carried in the args the cascade reads too, so a partitioned producer + // records the same identity everything else propagates. + let partition = job + .args + .as_ref() + .and_then(|a| a.0.get(windmill_common::partition::PARTITION_ARG)) + .and_then(|v| serde_json::from_str::(v.get()).ok()) + .unwrap_or_else(|| UNPARTITIONED.to_string()); + let (status, error) = match result { + Ok(_) => (MaterializationStatus::Materialized, None), + Err(e) => (MaterializationStatus::Failed, Some(e.to_string())), + }; + let recorded = match conn { + Connection::Sql(db) => windmill_common::materialization::record_materialization( + db, + &job.workspace_id, + windmill_common::assets::AssetKind::Dbt, + &m.target_path, + &partition, + status, + None, + None, + Some(job.id), + error.as_deref(), + ) + .await + .map_err(|e| anyhow::anyhow!("{e:#}")), + Connection::Http(client) => { + crate::agent_workers::record_materialization_from_agent_http( + client, + &job.workspace_id, + &RecordMaterializationRequest { + asset_kind: windmill_common::assets::AssetKind::Dbt, + asset_path: m.target_path.clone(), + partition, + status, + snapshot_id: None, + row_count: None, + job_id: Some(job.id), + error, + schema: None, + }, + ) + .await + } + }; + if let Err(e) = recorded { + tracing::warn!( + "recording the materialization of dbt://{} failed: {e:#}", + m.target_path + ); + } } /// True when `path` contains only `Normal`/`CurDir` components, i.e. it cannot diff --git a/docs/dbt-runtime.md b/docs/dbt-runtime.md index dc06a22fa0..3749bda3ea 100644 --- a/docs/dbt-runtime.md +++ b/docs/dbt-runtime.md @@ -47,6 +47,7 @@ the dominant way dbt is orchestrated today. | 22 | Naming | Match Cosmos field names; importer deferred | | 23 | Descriptor | `wm_dbt.yaml` inside the project, OPTIONAL. See below | | 24 | Warehouse | Configured on the workspace by name, `main` by default. See below | +| 25 | Cascade direction | Into a relation, not out of a run: `// materialize manual dbt://…` declares a write from any language but dbt's own and wakes `# on dbt://…` subscribers; a finished dbt run still does not dispatch. See "No cascade *from* dbt" | ## Decision 1: engine toggle, and why the shipped default is not Fusion yet @@ -157,11 +158,15 @@ build and an enterprise build whose key did not verify. workspace warehouse's NAME, so two scripts running against the same warehouse agree on identity. -The SCHEME names the producer, because dbt is the only thing that creates one of -these: no other language derives warehouse relations, `// materialize` takes -DuckLake targets only, and a dbt run does not dispatch. Calling the kind -something generic promised a parity with native Snowflake and BigQuery scripts -that does not exist. +The SCHEME names the namespace dbt made, not an exclusive producer. dbt is what +put warehouse relations in the asset graph and is what derives them from a +project; no other language *infers* one, and calling the kind something generic +promised a parity with native Snowflake and BigQuery scripts that does not exist. +A script can nonetheless DECLARE that it writes one — `// materialize manual +dbt:////`, in any language but dbt's own, whose writes +come from its manifest — and that declaration lands on the same node the dbt model +reading the relation does, because identity is the relation rather than the tool. +See "No cascade *from* dbt" below. The PATH is the physical relation, and that is the load-bearing half. dbt-core has no cross-project `ref()`: two projects meet when one materializes a mart and @@ -648,10 +653,13 @@ Two consequences worth knowing: dropped would be filtered out of its own run's graph. The pinned version's nodes are the scope instead. -## No cascade from dbt, and no pipeline membership +## No cascade *from* dbt, and no pipeline membership A finished dbt run does not trigger anything. Its models are recorded, drawn and -tracked; they do not fan out. +tracked; they do not fan out. The opposite direction does: a script that declares +`// materialize manual dbt:////` is an ordinary producer +of that relation, and its completion wakes `# on dbt://` subscribers +through the same fan-out every other asset kind uses. A dbt script is also not a pipeline member (`in_pipeline` is forced false for `ScriptLang::Dbt` at deploy). It materializes warehouse tables, so it looks like @@ -663,22 +671,99 @@ Its models are `dbt://` assets in the shared graph regardless: that is what puts a native script reading one of them on the same node, and it is independent of pipeline membership. -dbt already orders its own DAG, so a cascade would only ever add one thing: -waking a Windmill script that reads a mart. That edge is real but narrow, and -only half of it exists — nothing outside dbt can declare a `dbt://` write -(`// materialize` accepts DuckLake targets only), so the reverse direction, an -ingestion script waking a dbt project, cannot be expressed at all. - -Against that, dispatching correctly from dbt is not cheap. A run's `select` can -build any subset of the project, so the deploy-time write set is not what ran; -using it wakes consumers of relations the run never touched, and narrowing it -needs a per-job record of what was built, which the per-relation state table -cannot supply (it keeps one row per relation, stamped with the last writer). +dbt already orders its own DAG, so a cascade out of a run would only ever add one +thing: waking a Windmill script that reads a mart. That edge is real but narrow, +and dispatching it correctly is not cheap. A run's `select` can build any subset +of the project, so the deploy-time write set is not what ran; using it wakes +consumers of relations the run never touched, and narrowing it needs a per-job +record of what was built, which the per-relation state table cannot supply (it +keeps one row per relation, stamped with the last writer). So dbt materializes and reports, and `asset_dispatch` returns early for -`ScriptLang::Dbt`. A `# on dbt://` subscription is refused outright at -deploy rather than accepted and left dormant — an edge drawn on the canvas that -can never fire is worse than an error saying so. +`ScriptLang::Dbt`. Wiring it up later means deciding what a selective run should +notify — that decision is the work, not the plumbing. + +### Declaring the write, and which subscriptions are refused + +`// materialize manual dbt:////` is how an ingestion +script says it writes a warehouse relation. `manual` is not a mode but the only +mode: nothing generates warehouse DDL, so the script issues its own write and +Windmill records the outcome — the same `materialized_partition` row a DuckLake +target lands, so the relation carries a last writer on the run page and the graph. +It is language-agnostic (the DuckLake write ENGINE is DuckDB's; this declaration +is anyone's but a dbt project's, whose writes are read from its manifest), and the +recording happens in the generic job path +(`record_declared_warehouse_write`) rather than in an executor, for the same +reason. Identity is unchanged — the physical relation — so the ingestion script +and the dbt model reading it are one node, and a `source` declared on the relation +puts the whole thing on one lineage. `// data_test` is refused beside it: those +checks are probes the DuckDB executor splices around a managed write, so on a +warehouse relation — which the script writes itself, from any language — nothing +would run them, and a declarer would deploy green with its assertions silently +skipped. Assert on the relation with a dbt test in the project that reads it. +The `` segment is resolved at +deploy for the same reason a descriptor's `profile.warehouse` is: a name no +warehouse answers to is not a namespace, it strands the write on a node nothing +else reaches. + +Known boundary, shared with every other runtime pipeline annotation: the record +is written from the normal execution path, and recording and cascading are decided +separately, so the routes off it differ. + +* A **dedicated worker** never enters that path — it bypasses the record exactly + as it bypasses `// partitioned` resolution — while its job is still a top-level + `Script`, so the fan-out (which reads the deploy-time `asset` rows) runs. It + cascades and records nothing, leaving the relation with no last writer. +* A **flow runner** bypasses the path too, and is routed by `flow_step_id`, which + `is_eligible_kind` rejects. Neither record nor cascade. +* A **flow step running a deployed script** enters the path as a `Script` job, so + it records — and carries a `flow_step_id`, so it never cascades. +* A **flow step with an inline body** is a `FlowScript` job, which the recording + guard excludes along with previews: neither. + +Fixing the recording half is one change for every runtime pipeline annotation, +not this one. + +A `# on dbt://` subscription is held to the same relation a producer +is — a whole `//` under a configured warehouse, checked +by the validator the `// materialize` target goes through, since two spellings of +that rule would refuse and accept the same string. Beyond that it is refused in +exactly one shape: when every script that writes that relation is a dbt one. Nothing +produces it yet is NOT that shape — a subscriber may be deployed before its +producer, as for every other asset kind, and refusing there would break +deploy-order-independent syncs. A dbt script may neither subscribe nor declare a +`// materialize`: its graph ingest republishes that path's trigger and asset rows +wholesale, so either annotation would deploy something the dependency job then +silently removes — while the declared write would still stamp the relation on +every run. + +The producer set is read as it stands committed, minus the deploying script's own +rows — those describe the version being replaced, so a script dropping its +`// materialize` while adding a subscription would otherwise count itself as the +producer that wakes it, which it could not be anyway (the dispatcher skips +self-loops). + +What that leaves is a subscription accepted while it was live and later orphaned. +A dbt project that claims the relation afterwards names those edges in its own log +rather than leaving them silently dormant — the same "an edge that can never fire +is worse than saying so" the refusal is for, at the other point where it is +knowable. Both points that publish ownership warn: the deploy, and a run whose +static descriptor found its profile moved. An agent run publishes none — it is +forced to per-run models, so it stores a job-pinned snapshot and leaves workspace +ownership with the deployed graph — so it cannot orphan a subscription either. + +Two orphanings are reported nowhere, and both are accepted rather than overlooked. +A native producer that drops its `// materialize` and leaves dbt alone on the +relation: the deploy that causes it does not touch the subscriber. And the +interleaving where a dbt ingest commits between a subscriber's producer check and +its own commit — the check sees no producer and accepts, the ingest's warning +query sees no trigger and says nothing. Closing the second means a per-relation +lock shared by the deploy path and the ingest, and the ingest takes +`script … FOR UPDATE` before its own advisory lock, so a deploy holding relation +locks first inverts that order into a deadlock across two subsystems — a worse +failure than the cosmetic edge it would prevent. Both are bounded the same way: +the next deploy of that project warns, and the canvas is where they show +meanwhile. A plain READ still renders the consumer beside the model, which is what makes the lineage one graph — but it is written in the script's own code, not in a @@ -687,8 +772,8 @@ comment: the body parsers resolve an asset URI from a string literal Python, TS/Bun/Deno, DuckDB or Ansible script is the read. Those four are the languages with a body-asset parser; the native warehouse ones (snowflake, bigquery, postgresql, mysql, mssql) declare no assets at all today, so a mart -they consume joins the graph only once that inference exists. Wiring the trigger up later means deciding what a -selective run should notify — that decision is the work, not the plumbing. +they consume joins the graph only once that inference exists — while a relation +one of them WRITES joins it now, through the annotation. ## Live per-model progress, and why only dbt-core 1.x has it @@ -1215,17 +1300,22 @@ Against a real dbt project (jaffle_shop shape) and the local Postgres: script reading one of the marts gets an edge to it. 6. **Shared node**: a native script that READS a mart renders as a reader of the same node the dbt model writes — one node, not two islands. Declared with a - plain read (`# dbt://`), never `# on`: a `dbt://` subscription is - refused at deploy, because nothing but dbt writes a warehouse relation and a - dbt run does not dispatch (see "no cascade from dbt"). -7. **Selection**: descriptor `select`/`exclude`, and a run-arg override, each + plain read (`# dbt://`), never `# on`: a subscription to a relation dbt + alone builds is refused at deploy, since a dbt run does not dispatch. +7. **Declared write**: a native `// materialize manual dbt://` script + and a dbt project reading that relation as a `source` render as one node; a + run of the script records its materialization and wakes a + `# on dbt://` subscriber — a subscription only that producer makes + wakeable, the dbt project reading the relation being no producer of it (see + "no cascade *from* dbt"). +8. **Selection**: descriptor `select`/`exclude`, and a run-arg override, each build only the expected subset. -8. **Dynamic descriptors**: a `{{ }}` placeholder in `vars` re-ingests the graph +9. **Dynamic descriptors**: a `{{ }}` placeholder in `vars` re-ingests the graph from the run's own manifest, so a model that placeholder enables appears in the same run that builds it. -9. **Both credential paths**: resource-rendered `profiles.yml`, and the project's +10. **Both credential paths**: resource-rendered `profiles.yml`, and the project's own `profiles.yml` with env-var injection. -10. **Caching**: a second run reuses the cached `dbt_packages/` with no network +11. **Caching**: a second run reuses the cached `dbt_packages/` with no network fetch. Keep only tests that pin behavior a future change could break. Per AGENTS.md, diff --git a/docs/ducklake-materialization.md b/docs/ducklake-materialization.md index 32fff7ca23..834b48c211 100644 --- a/docs/ducklake-materialization.md +++ b/docs/ducklake-materialization.md @@ -606,8 +606,10 @@ how-to (extract-engine choice, cursor recipes, schema-drift handling, worked examples) lives in windmilldocs `core_concepts/63_pipelines` → "Ingestion (EL)"; this section records only what future feature work must not break. -- **`// materialize` is DuckDB-only** (deploy-rejected elsewhere, managed and - `manual` alike — `windmill-api-scripts/src/scripts.rs`), and the SDK +- **A `ducklake://` `// materialize` is DuckDB-only** (deploy-rejected + elsewhere, managed and `manual` alike — `windmill-api-scripts/src/scripts.rs`; + a `dbt://` warehouse-relation target is the one any language but dbt's own may + declare, and it is track-only — see `docs/dbt-runtime.md`), and the SDK materialize helpers (`upsert_partition` / `upsertPartition`) build their SQL inside the SDK, so the asset parsers cannot see the write. A polyglot node that "writes the lake directly" therefore deploys with **no output edge** — diff --git a/frontend/src/lib/components/assets/AssetGraph/resolveGraph.test.ts b/frontend/src/lib/components/assets/AssetGraph/resolveGraph.test.ts index de00aef2be..c57687d78b 100644 --- a/frontend/src/lib/components/assets/AssetGraph/resolveGraph.test.ts +++ b/frontend/src/lib/components/assets/AssetGraph/resolveGraph.test.ts @@ -319,6 +319,18 @@ describe('resolveGraph', () => { expect(assetTrigKeys(r, 'f/x/open')).toEqual(['ducklake:main.orders']) }) + // A relation a native `// materialize manual dbt://…` script writes wakes its + // subscribers, so hiding the edge would leave the author's own annotation off + // the canvas. The deploy refuses the ones that cannot fire. + it('draws an explicit dbt:// subscription as an unsaved trigger overlay', () => { + const liveAnnotations = { + scriptPath: 'f/x/open', + annotations: ann({ triggerAssets: [{ kind: 'dbt', path: 'main/analytics/orders' }] }) + } + const r = resolveGraph(input({ liveAnnotations })) + expect(assetTrigKeys(r, 'f/x/open')).toEqual(['dbt:main/analytics/orders']) + }) + it('open-script live annotations add unsaved triggers, deduped vs persisted', () => { const base = baseGraph({ triggers: [ diff --git a/frontend/src/lib/components/assets/AssetGraph/resolveGraph.ts b/frontend/src/lib/components/assets/AssetGraph/resolveGraph.ts index 867fa6f036..5c8aa697a4 100644 --- a/frontend/src/lib/components/assets/AssetGraph/resolveGraph.ts +++ b/frontend/src/lib/components/assets/AssetGraph/resolveGraph.ts @@ -77,23 +77,15 @@ function persistedNativeKinds(base: AssetGraphResponse, path: string): Set = new Set(['ducklake', 's3object']) -/** Whether a subscription on this kind can ever fire once deployed. - * - * A `dbt://` one cannot: dbt is the only producer of a warehouse relation and - * a dbt run does not dispatch, so the deploy refuses `// on dbt://…` outright - * (`scripts.rs`). The editor must not draw an arrow the deploy will reject — - * applied to the EXPLICIT overlays; auto-derivation is already scoped by - * `AUTO_TRIGGER_KINDS`. Parsing is left alone so the Rust-parity test still - * compares like for like. */ -function canTrigger(kind: AssetKind): boolean { - return kind !== 'dbt' -} - /** `kind:path` refs of a script's `// materialize` write target(s) (base + * the scd2 `_current` companion), which the body `SELECT` doesn't express. */ function materializeWriteRefs(parsed: PipelineAnnotations): string[] { @@ -381,7 +373,7 @@ function makeContext(input: ResolveGraphInput): ResolveContext { const liveRefKeys = new Set() if (openIsSavedEdit) { if (liveAnnotations.scriptPath === openPath) { - for (const a of liveAnnotations.annotations.triggerAssets.filter((a) => canTrigger(a.kind))) + for (const a of liveAnnotations.annotations.triggerAssets) liveRefKeys.add(`${a.kind}:${a.path}`) // The `// materialize ` target is a declared *output*, but it // lives in an annotation (not the SQL body), so neither triggerAssets @@ -625,7 +617,7 @@ function seedDraftOverlays(acc: Accumulator, input: ResolveGraphInput) { // stable when the user clicks off this draft. Live annotations // (below) take over for the currently-open draft so keystroke // edits still update in real time. - for (const a of parsed.triggerAssets.filter((a) => canTrigger(a.kind))) { + for (const a of parsed.triggerAssets) { extraTriggers.push({ trigger_kind: 'asset', asset_kind: a.kind, @@ -704,7 +696,7 @@ function applyLiveBufferOverlay(acc: Accumulator, input: ResolveGraphInput, ctx: for (let i = extraTriggers.length - 1; i >= 0; i--) { if (extraTriggers[i].runnable_path === livePath) extraTriggers.splice(i, 1) } - for (const a of liveAnnotations.annotations.triggerAssets.filter((a) => canTrigger(a.kind))) { + for (const a of liveAnnotations.annotations.triggerAssets) { const key = `${a.kind}:${a.path}` if (assetKeys.has(key)) continue extraTriggers.push({ diff --git a/frontend/src/lib/components/assets/lib.ts b/frontend/src/lib/components/assets/lib.ts index 23531d71f2..93367a725e 100644 --- a/frontend/src/lib/components/assets/lib.ts +++ b/frontend/src/lib/components/assets/lib.ts @@ -96,10 +96,11 @@ export function formatAssetKind(asset: { case 'volume': return 'Volume' case 'dbt': - // The SCHEME says dbt because dbt is the only thing that creates one; - // the PATH stays the relation, so a mart one project builds and the - // `source` the next reads land on one node — their dbt `unique_id`s - // differ where the relation does not (docs/dbt-runtime.md, decision 11). + // The SCHEME says dbt because dbt is what derives these relations; the + // PATH stays the relation, so a mart one project builds, the `source` + // the next reads, and a native `// materialize manual dbt://…` writer + // land on one node — their dbt `unique_id`s differ where the relation + // does not (docs/dbt-runtime.md, decision 11). return 'dbt table' } } diff --git a/frontend/src/lib/components/copilot/chat/pipeline/core.ts b/frontend/src/lib/components/copilot/chat/pipeline/core.ts index 7583c1b356..f696b1c869 100644 --- a/frontend/src/lib/components/copilot/chat/pipeline/core.ts +++ b/frontend/src/lib/components/copilot/chat/pipeline/core.ts @@ -338,8 +338,8 @@ export function getPipelinePromptSection(ctx: PipelineContext): string { Data Pipeline editor (ACTIVE): - The user has the /pipeline/${ctx.folder} editor open. A pipeline is a DAG of scripts (nodes) connected by storage assets (DuckLake tables, data tables, S3 objects, volumes, resources) and execution triggers. - Annotations are top-of-file comments in the NODE'S OWN comment syntax: \`--\` for SQL (duckdb/postgresql), \`#\` for python3/bash, \`//\` for bun/TS. The \`//\` shown below is the TS form — translate it (a \`// pipeline\` line in a SQL node is a syntax error that won't deploy). -- A script becomes a pipeline node when its source starts with the \`// pipeline\` annotation. Declare execution-DAG inputs with \`// on \` (e.g. \`// on ducklake://main/orders\`). Outputs are inferred from what the body writes (wmill SDK calls / SQL CREATE TABLE / writeS3File); declare a managed output with \`// materialize \`. Optional badges: \`// partitioned \`, \`// freshness \`, \`// tag \`, \`// retry [delay]\`, \`// data_test ...\`, \`// measure = [where ]\`, \`// dimension = \`. -- \`materialize\` (the managed output): \`// materialize \` means the runtime writes the node's output table FOR you — write the body as a single SELECT and the runtime wraps it in the create/replace, so do NOT also write your own CREATE TABLE / INSERT. IMPORTANT: \`// materialize\` is **DuckDB-only** and its target MUST be a DuckLake table (\`ducklake:///
\`) — deploy rejects it on any other language or target. For a \`python3\`/\`bun\`/\`postgresql\` node, do NOT use \`// materialize\`; write the output via the SDK instead (e.g. \`wmill.writeS3File(...)\`, a \`CREATE TABLE\` in postgresql, or \`wmill.databaseUrlFromResource\`/ducklake helpers) and let the output be inferred. Reach for \`duckdb\` when a node should materialize a DuckLake table. Write strategy: with no option it REPLACES the whole table each run (full refresh; the only mode whose output columns may change); \`// materialize append\` INSERT-appends rows (incremental); \`// materialize key=\` merges/upserts on \`\`. \`// materialize manual \` opts OUT of managed writes — the script writes its own DDL and the annotation only records the output asset for lineage. \`materialize\` is paired with partitioning for incremental pipelines: a \`// partitioned \` node runs once per partition (append/merge into a fixed-schema table), and the \`{partition}\` token — usable in any asset URI AND in the body SQL — is substituted with the current partition's IDENTITY string at run time. To filter the source to the active slice on a time grain, use the runtime-injected macro: \`WHERE wm_partition() = {partition}\`. \`wm_partition(ts)\` buckets a timestamp with the exact identity format the runtime used (daily/hourly/weekly/monthly), so it always matches and you never hand-write a \`strftime\` format. Do NOT write \`= TIMESTAMP {partition}\`: the identity string is not a valid timestamp literal for hourly/weekly/monthly and errors at runtime. For \`dynamic\` partitioning the identity is your caller-supplied key (not a timestamp, no macro), so filter on it directly: \`WHERE = {partition}\`. \`materialize\` is an output DECLARATION on the node — it is not a command; there is no "materialize run". +- A script becomes a pipeline node when its source starts with the \`// pipeline\` annotation. Declare execution-DAG inputs with \`// on \` (e.g. \`// on ducklake://main/orders\`). Outputs are inferred from what the body writes (wmill SDK calls / SQL CREATE TABLE / writeS3File); declare a managed output with \`// materialize \`. Optional badges: \`// partitioned \`, \`// freshness \`, \`// tag \`, \`// retry [delay]\`, \`// data_test ...\` (managed DuckLake targets only — deploy rejects it beside a \`dbt://\` target), \`// measure = [where ]\`, \`// dimension = \`. +- \`materialize\` (the managed output): a managed \`// materialize ducklake:///
\` means the runtime writes the node's output table FOR you — write the body as a single SELECT and the runtime wraps it in the create/replace, so do NOT also write your own CREATE TABLE / INSERT. The \`dbt://\` target below is the opposite: the node writes its own DDL and none of the write strategies apply to it. IMPORTANT: a MANAGED \`// materialize\` is **DuckDB-only** and its target MUST be a DuckLake table (\`ducklake:///
\`) — deploy rejects a \`ducklake://\` target on any other language. For a \`python3\`/\`bun\`/\`postgresql\` node writing the lake, do NOT use \`// materialize\`; write the output via the SDK instead (e.g. \`wmill.writeS3File(...)\`, a \`CREATE TABLE\` in postgresql, or \`wmill.databaseUrlFromResource\`/ducklake helpers) and let the output be inferred. Reach for \`duckdb\` when a node should materialize a DuckLake table. The one target any language BUT DBT'S OWN may declare (a dbt project's writes come from its manifest, so \`// materialize\` on a dbt script is rejected at deploy) is a WAREHOUSE RELATION: \`// materialize manual dbt:////\`, with \`\` a warehouse the workspace configures under Settings → dbt. \`manual\` is its only mode — nothing generates warehouse DDL, so the node issues its own write and the annotation records the outcome. Use it on an ingestion node a dbt project reads as a \`source\`: the declared relation and the dbt model become ONE graph node, and a downstream \`// on dbt:////\` fires when that node completes. Write strategy: with no option it REPLACES the whole table each run (full refresh; the only mode whose output columns may change); \`// materialize append\` INSERT-appends rows (incremental); \`// materialize key=\` merges/upserts on \`\`. \`// materialize manual \` opts OUT of managed writes — the script writes its own DDL and the annotation only records the output asset for lineage. \`materialize\` is paired with partitioning for incremental pipelines: a \`// partitioned \` node runs once per partition (append/merge into a fixed-schema table), and the \`{partition}\` token — usable in any asset URI AND in the body SQL — is substituted with the current partition's IDENTITY string at run time. To filter the source to the active slice on a time grain, use the runtime-injected macro: \`WHERE wm_partition() = {partition}\`. \`wm_partition(ts)\` buckets a timestamp with the exact identity format the runtime used (daily/hourly/weekly/monthly), so it always matches and you never hand-write a \`strftime\` format. Do NOT write \`= TIMESTAMP {partition}\`: the identity string is not a valid timestamp literal for hourly/weekly/monthly and errors at runtime. For \`dynamic\` partitioning the identity is your caller-supplied key (not a timestamp, no macro), so filter on it directly: \`WHERE = {partition}\`. \`materialize\` is an output DECLARATION on the node — it is not a command; there is no "materialize run". - \`measure\` / \`dimension\` (declared metrics): on a node that materializes a DuckLake table, \`// measure = [where ]\` names the canonical way to aggregate that table (e.g. \`// measure revenue = sum(amount) where not is_refund\`), and \`// dimension = \` names a way to slice it (e.g. \`// dimension region = region\`, \`// dimension month = date_trunc('month', ordered_at)\`). They execute nothing: they are catalogued at deploy so the editor and other agents can reuse the definition instead of re-deriving it and silently disagreeing. Keep the predicate in the \`where\` clause rather than folding it into the aggregate: it is rendered as \` FILTER (WHERE )\`, which is what lets two measures with different predicates sit under one GROUP BY. DuckLake-only, and only meaningful next to \`// materialize\`. Declare one when a number carries a judgement call someone else would get wrong (refunds excluded, test rows dropped, which column is the amount); do NOT blanket every table with measures, an obvious \`count(*)\` earns nothing. To USE a metric another node declares, read that node with read_pipeline_node and reuse its exact expression rather than guessing it. - Use get_pipeline_graph to see the current nodes/assets/triggers, and read_pipeline_node before editing one. - Every node of this pipeline lives at \`f/${ctx.folder}/\` — \`${ctx.folder}\` is the folder name and \`f/\` is the owner prefix every workspace path carries, so write it exactly once (never \`f/f/…\`, and never a bare \`\`). diff --git a/system_prompts/auto-generated/prompts.ts b/system_prompts/auto-generated/prompts.ts index 1feeadc162..5776fd6d0d 100644 --- a/system_prompts/auto-generated/prompts.ts +++ b/system_prompts/auto-generated/prompts.ts @@ -984,7 +984,7 @@ A script joins the pipeline when its source begins with the \`pipeline\` annotat SELECT * FROM read_csv($file) \`\`\` - **Outputs** are inferred from what the body writes — a \`CREATE TABLE\`, a \`wmill.writeS3File(...)\`, a DuckLake/datatable write. To declare a managed output explicitly, use \`// materialize \`. -- Optional badges: \`// partitioned \`, \`// freshness \` (e.g. \`1h\`), \`// tag \`, \`// retry [delay]\`, \`// data_test ...\`. +- Optional badges: \`// partitioned \`, \`// freshness \` (e.g. \`1h\`), \`// tag \`, \`// retry [delay]\`, \`// data_test ...\` (managed DuckLake targets only — deploy rejects it beside a \`dbt://\` one). ## S3 object wiring (storage form matters) @@ -999,9 +999,11 @@ The key must be a **string literal** — the graph parser is static and cannot f ## Materialize (the managed output) -> **\`// materialize\` is DuckDB-only**, and its target must be a DuckLake table (\`ducklake:///
\`). Deploy **rejects** \`// materialize\` on any other language (\`python3\`, \`bun\`, \`postgresql\`) or a non-DuckLake target. For a non-DuckDB node, do **not** use \`// materialize\` — write the output via the SDK (\`wmill.writeS3File(...)\`, a postgresql \`CREATE TABLE\`, ducklake helpers, …) and let it be inferred. Use \`duckdb\` when a node should materialize a DuckLake table. +> **A MANAGED \`// materialize\` is DuckDB-only**, and its target must be a DuckLake table (\`ducklake:///
\`). Deploy **rejects** a \`ducklake://\` \`// materialize\` on any other language (\`python3\`, \`bun\`, \`postgresql\`). For a non-DuckDB node writing the lake, do **not** use \`// materialize\` — write the output via the SDK (\`wmill.writeS3File(...)\`, ducklake helpers, …) and let it be inferred. Use \`duckdb\` when a node should materialize a DuckLake table. +> +> The one target ANY language may declare (except a dbt script, whose writes come from its manifest) is a **warehouse relation**: \`// materialize manual dbt:////\`, where \`\` is a warehouse the workspace configures under Settings → dbt. \`manual\` is the only mode it has — nothing generates warehouse DDL, so the node issues its own write (a postgresql \`CREATE TABLE\` / \`INSERT\`, an SDK load, …) and the annotation records the outcome. Use it on an ingestion node whose output a dbt project reads as a \`source\`: the declared relation and the dbt model land on ONE graph node, and a downstream \`// on dbt:////\` fires when the ingestion node completes. -\`// materialize \` tells the runtime to write the node's output table **for you**: write the body as a single \`SELECT\` and the runtime wraps it in the create/replace — do **not** also write your own \`CREATE TABLE\` / \`INSERT\`. Write strategy: +A managed \`// materialize ducklake:///
\` tells the runtime to write the node's output table **for you**: write the body as a single \`SELECT\` and the runtime wraps it in the create/replace — do **not** also write your own \`CREATE TABLE\` / \`INSERT\`. (The opposite holds for the \`dbt://\` target above: there the node writes its own DDL and the strategies below do not apply.) Write strategy: - no option → **replace** the whole table each run (full refresh; the only mode whose output columns may change); - \`// materialize append\` → INSERT-append rows (incremental); diff --git a/system_prompts/base/pipeline-base.md b/system_prompts/base/pipeline-base.md index b2b428c401..3f2d985de4 100644 --- a/system_prompts/base/pipeline-base.md +++ b/system_prompts/base/pipeline-base.md @@ -37,7 +37,7 @@ A script joins the pipeline when its source begins with the `pipeline` annotatio SELECT * FROM read_csv($file) ``` - **Outputs** are inferred from what the body writes — a `CREATE TABLE`, a `wmill.writeS3File(...)`, a DuckLake/datatable write. To declare a managed output explicitly, use `// materialize `. -- Optional badges: `// partitioned `, `// freshness ` (e.g. `1h`), `// tag `, `// retry [delay]`, `// data_test ...`. +- Optional badges: `// partitioned `, `// freshness ` (e.g. `1h`), `// tag `, `// retry [delay]`, `// data_test ...` (managed DuckLake targets only — deploy rejects it beside a `dbt://` one). ## S3 object wiring (storage form matters) @@ -52,9 +52,11 @@ The key must be a **string literal** — the graph parser is static and cannot f ## Materialize (the managed output) -> **`// materialize` is DuckDB-only**, and its target must be a DuckLake table (`ducklake:///
`). Deploy **rejects** `// materialize` on any other language (`python3`, `bun`, `postgresql`) or a non-DuckLake target. For a non-DuckDB node, do **not** use `// materialize` — write the output via the SDK (`wmill.writeS3File(...)`, a postgresql `CREATE TABLE`, ducklake helpers, …) and let it be inferred. Use `duckdb` when a node should materialize a DuckLake table. +> **A MANAGED `// materialize` is DuckDB-only**, and its target must be a DuckLake table (`ducklake:///
`). Deploy **rejects** a `ducklake://` `// materialize` on any other language (`python3`, `bun`, `postgresql`). For a non-DuckDB node writing the lake, do **not** use `// materialize` — write the output via the SDK (`wmill.writeS3File(...)`, ducklake helpers, …) and let it be inferred. Use `duckdb` when a node should materialize a DuckLake table. +> +> The one target ANY language may declare (except a dbt script, whose writes come from its manifest) is a **warehouse relation**: `// materialize manual dbt:////`, where `` is a warehouse the workspace configures under Settings → dbt. `manual` is the only mode it has — nothing generates warehouse DDL, so the node issues its own write (a postgresql `CREATE TABLE` / `INSERT`, an SDK load, …) and the annotation records the outcome. Use it on an ingestion node whose output a dbt project reads as a `source`: the declared relation and the dbt model land on ONE graph node, and a downstream `// on dbt:////` fires when the ingestion node completes. -`// materialize ` tells the runtime to write the node's output table **for you**: write the body as a single `SELECT` and the runtime wraps it in the create/replace — do **not** also write your own `CREATE TABLE` / `INSERT`. Write strategy: +A managed `// materialize ducklake:///
` tells the runtime to write the node's output table **for you**: write the body as a single `SELECT` and the runtime wraps it in the create/replace — do **not** also write your own `CREATE TABLE` / `INSERT`. (The opposite holds for the `dbt://` target above: there the node writes its own DDL and the strategies below do not apply.) Write strategy: - no option → **replace** the whole table each run (full refresh; the only mode whose output columns may change); - `// materialize append` → INSERT-append rows (incremental); From 8f553eab353103fd8a28a00532e1766f133590de Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 7 Sep 2026 15:38:38 +0000 Subject: [PATCH 008/144] fix: point the app viewer's edit button at the editor for the app's kind (#11009) Claude-Session: https://claude.ai/code/session_01GDiZaPzhC4R9G4hLPgy1B2 Co-authored-by: Claude Opus 5 (1M context) --- .../apps/editor/InWorkspaceAppViewer.svelte | 16 +++++++++++----- .../components/search/GlobalSearchModal.svelte | 2 +- .../(logged)/apps/get/[...path]/+page.svelte | 10 +++++----- .../(logged)/apps_raw/get/[...path]/+page.svelte | 3 +-- 4 files changed, 18 insertions(+), 13 deletions(-) diff --git a/frontend/src/lib/components/apps/editor/InWorkspaceAppViewer.svelte b/frontend/src/lib/components/apps/editor/InWorkspaceAppViewer.svelte index d5839c6766..99ea07c8fd 100644 --- a/frontend/src/lib/components/apps/editor/InWorkspaceAppViewer.svelte +++ b/frontend/src/lib/components/apps/editor/InWorkspaceAppViewer.svelte @@ -22,19 +22,22 @@ let { workspace, - path, - editHref + path }: { workspace: string path: string - /** Where the Edit button points (low-code vs raw editor). */ - editHref: string } = $props() let app: any = $state(undefined) let notExists = $state(false) let noPermission = $state(false) let canWriteApp = $state(false) + /** Raw vs low-code, read from the app itself rather than from the route: + * both kinds render here and either route serves either kind (links to a raw + * app point at /apps/get all over the app), so only the app can say which + * editor the Edit button must open. */ + let isRawApp = $state(false) + let editHref = $derived(`${base}/${isRawApp ? 'apps_raw' : 'apps'}/edit/${path}?nodraft=true`) let refresh: (() => void) | undefined // The opaque iframe loads the dedicated cookieless, chrome-less viewer route. @@ -103,11 +106,14 @@ } } - // Edit button: determine write access on this real-origin page (cookie). + // Edit button: determine write access and which editor to open on this + // real-origin page (cookie). The sandboxed low-code app never loads on this + // page (it loads inside the opaque iframe), so `app` can't be the source. async function loadPerms() { try { const lite: any = await AppService.getAppLiteByPath({ workspace, path }) canWriteApp = canWrite(lite?.path, lite?.extra_perms ?? {}, $userStore) + isRawApp = !!lite?.raw_app } catch (_) { canWriteApp = false } diff --git a/frontend/src/lib/components/search/GlobalSearchModal.svelte b/frontend/src/lib/components/search/GlobalSearchModal.svelte index a117d2769c..084ad9d492 100644 --- a/frontend/src/lib/components/search/GlobalSearchModal.svelte +++ b/frontend/src/lib/components/search/GlobalSearchModal.svelte @@ -427,7 +427,7 @@ path = `/apps/get/${e.path}` break case 'raw_app': - path = `/raw_apps/get/${e.path}` + path = `/apps_raw/get/${e.path}` break default: path = '/' diff --git a/frontend/src/routes/(root)/(logged)/apps/get/[...path]/+page.svelte b/frontend/src/routes/(root)/(logged)/apps/get/[...path]/+page.svelte index 0af3cdeded..4ff8670cda 100644 --- a/frontend/src/routes/(root)/(logged)/apps/get/[...path]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/apps/get/[...path]/+page.svelte @@ -1,11 +1,11 @@ + + dispatch('canceled')} + on:confirmed={() => dispatch('confirmed', { selectedTriggers, selectedAgents })} +> +
+ {#if draftTriggers.length > 0} +
+
+ {`Your ${runnable} has draft triggers. Select which draft triggers to deploy with the ${runnable}. Undeployed draft triggers will be permanently deleted.`} +
+ +
5 ? 'h-[300px]' : ''}> + +
+ + + + + + + {#each draftTriggers as trigger} + {@const SvelteComponent = triggerIconMap[trigger.type]} + {@const permission = checkSavePermissions(trigger)} + {@const isSelectedTrigger = isSelected(selectedTriggers, trigger)} + + + + + + {/each} + + + + + {/if} + + {#if draftAgents.length > 0} +
+
+ Saved agents this flow uses have unsaved changes. Select which ones to deploy with the + flow. An agent kept as a draft stays editable, and the flow runs the agent as currently + deployed. +
+ +
5 ? 'h-[300px]' : ''}> + +
+ + + + + + + + {#each draftAgents as agent (agent.path)} + {@const permission = checkAgentPermissions(agent)} + {@const isSelectedAgent = selectedAgents.some((a) => a.path === agent.path)} + + + + + + + + {/each} + + + + + {/if} + + diff --git a/frontend/src/lib/components/common/confirmationModal/DraftTriggersConfirmationModal.svelte b/frontend/src/lib/components/common/confirmationModal/DraftTriggersConfirmationModal.svelte deleted file mode 100644 index cfebae0e9f..0000000000 --- a/frontend/src/lib/components/common/confirmationModal/DraftTriggersConfirmationModal.svelte +++ /dev/null @@ -1,168 +0,0 @@ - - - dispatch('canceled')} - on:confirmed={() => dispatch('confirmed', { selectedTriggers })} -> -
-
- {`${isFlow ? 'Your flow' : 'Your script'} has draft triggers. Select which draft triggers to deploy with the ${isFlow ? 'flow' : 'script'}. Undeployed - draft triggers will be permanently deleted.`} -
- -
5 ? 'h-[300px]' : ''}> - -
- - - - - - - {#each draftTriggers as trigger} - {@const SvelteComponent = triggerIconMap[trigger.type]} - {@const permission = checkSavePermissions(trigger)} - {@const isSelectedTrigger = isSelected(selectedTriggers, trigger)} - - - - - - {/each} - - {#if draftTriggers.length === 0} - - - - {/if} - - - - - diff --git a/frontend/src/lib/components/flows/agentDraft.svelte.ts b/frontend/src/lib/components/flows/agentDraft.svelte.ts index 5a6a67ccb8..d4f8fa35de 100644 --- a/frontend/src/lib/components/flows/agentDraft.svelte.ts +++ b/frontend/src/lib/components/flows/agentDraft.svelte.ts @@ -6,6 +6,7 @@ import { sendUserToast } from '$lib/toast' import { canWrite } from '$lib/utils' import { userStore } from '$lib/stores' import { getUserExt } from '$lib/user' +import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte' import { useTriggerDraftSync, type TriggerDraftSync } from '../triggers/useTriggerDraftSync.svelte' import { logReusableAgentUsage } from './agentTelemetry' import { @@ -84,6 +85,95 @@ export interface AgentResourceState { wsSpecific: boolean } +/** + * Why writing this draft to its resource would be refused, if it would. Shared with the flow's + * deploy dialog, which lists the drafts of the agents a flow links: an agent the editor's own + * Deploy button rejects must not be deployable from a flow either. + * + * `currentPath` is the path the draft is being deployed to; pass undefined where the caller has + * none of its own to compare against. + */ +export function agentDraftDeployRefusal( + state: AgentResourceState, + currentPath: string | undefined +): string | undefined { + // The editor offers only values, but a transform can arrive from a step that was forked + // before this existed, or from the generic resource editor: say so rather than writing it. + const transformValued = transformValuedBrainKeys(state.args) + if (transformValued.length > 0) { + const fields = transformValued.map((key) => AGENT_BRAIN_LABELS[key] ?? key) + const many = fields.length > 1 + return `${fields.join(', ')} ${many ? 'are' : 'is'} set to an expression or an AI-filled value, which a saved agent cannot store. Replace ${many ? 'them' : 'it'} with a plain value before deploying.` + } + // The resource endpoint takes any JSON, so nothing downstream stops an agent that cannot run: + // the worker needs a provider to call and rejects a tool whose name it cannot pass to the + // model. Deploying one would break every flow linking it, so it is refused here. + const blocked = agentConfigRunError(state.args) + if (blocked) { + return blocked + } + // Renaming is not the agent editor's to do: moving the resource leaves every step that links to + // it naming a path that no longer exists, and reconciling those is a feature of its own. A + // renamed path can still reach here, the generic editor writing the same draft row and offering + // a path field, so refuse it rather than performing half of a rename. + if (currentPath && state.path !== currentPath) { + return `This draft renames the agent to ${state.path}. Deploy it from the resource editor instead.` + } + // Only a draft naming another type: the load refuses a resource that is not an agent, while a + // draft the generic resource editor wrote names no type at all and inherits the loaded one. + if (state.resource_type && state.resource_type !== 'ai_agent') { + return `This draft is a ${state.resource_type} resource, not an agent.` + } + return undefined +} + +/** + * Write an agent to its resource from the state the editor holds, which can be ahead of the + * persisted draft row: the form stays editable while a deploy is in flight. Surfaces that deploy + * the row itself go through `deployDraft` instead. + * + * `notAnAgent` separates the one failure that invalidates the caller's whole view of the path, its + * holding something else now, from a write that merely failed. + */ +type AgentWriteResult = { ok: true } | { ok: false; error: string; notAnAgent?: true } + +async function writeAgentResource( + workspace: string, + state: AgentResourceState, + noDeployed: boolean +): Promise { + const body = { + path: state.path, + value: state.args, + description: state.description, + labels: state.labels, + ws_specific: state.wsSpecific + } + try { + if (noDeployed) { + // A create needs a type, and every caller proved this path is an agent before offering it. + await ResourceService.createResource({ + workspace, + requestBody: { ...body, resource_type: state.resource_type ?? 'ai_agent' } + }) + } else { + // The type the caller proved is as old as its own load, and an update carries no type of + // its own: were the path deleted and recreated as something else meanwhile, this write + // would put an agent config inside that resource. Reading it again narrows the window to + // the request rather than to however long the editor or the dialog stayed open. + const current = await ResourceService.getResource({ workspace, path: state.path }) + const refused = agentEditorRefusal(state.path, current.resource_type) + if (refused) { + return { ok: false, error: refused, notAnAgent: true } + } + await ResourceService.updateResource({ workspace, path: state.path, requestBody: body }) + } + } catch (err) { + return { ok: false, error: `Could not save agent: ${err}` } + } + return { ok: true } +} + export interface AgentDraftOptions { /** The `ai_agent` resource being edited. */ path: () => string | undefined @@ -219,6 +309,16 @@ export function useAgentDraft(opts: AgentDraftOptions): AgentDraftHandle { // config before the autosave lands. state = ((r as any).draft as AgentResourceState | undefined) ?? structuredClone(deployedState) + // Adopt the row's timestamp as this tab's baseline. Without it the first save from + // each tab goes out with no `last_sync`, which the backend treats as unconditional + // and so silently overwrites another tab's newer draft. It also clears any parked + // conflict or failure for the key: a conflict is deliberately sticky (the retry + // keeps the same baseline), and nothing else mounts a resolver for `resource` + // drafts, so re-opening the agent is the only place it can be resolved. + UserDraftDbSyncer.recordRemoteSync( + { workspace: ws, itemKind: 'resource', path }, + (r as { draft_saved_at?: string }).draft_saved_at + ) loading = false await sync.maybeRestore() }, @@ -237,42 +337,9 @@ export function useAgentDraft(opts: AgentDraftOptions): AgentDraftHandle { const ws = opts.workspace() const s = state if (!ws || !s) return false - // The editor offers only values, but a transform can arrive from a step that was forked - // before this existed, or from the generic resource editor: say so rather than writing it. - const transformValued = transformValuedBrainKeys(s.args) - if (transformValued.length > 0) { - const fields = transformValued.map((key) => AGENT_BRAIN_LABELS[key] ?? key) - const many = fields.length > 1 - sendUserToast( - `${fields.join(', ')} ${many ? 'are' : 'is'} set to an expression or an AI-filled value, which a saved agent cannot store. Replace ${many ? 'them' : 'it'} with a plain value before deploying.`, - true - ) - return false - } - // The resource endpoint takes any JSON, so nothing downstream stops an agent that cannot run: - // the worker needs a provider to call and rejects a tool whose name it cannot pass to the - // model. Deploying one would break every flow linking it, so it is refused here. - const blocked = agentConfigRunError(s.args) - if (blocked) { - sendUserToast(blocked, true) - return false - } - // Renaming is not this editor's to do: moving the resource leaves every step that links to it - // naming a path that no longer exists, and reconciling those is a feature of its own. A - // renamed path can still reach here, the generic editor writing the same draft row and - // offering a path field, so refuse it rather than performing half of a rename. - const currentPath = opts.path() - if (currentPath && s.path !== currentPath) { - sendUserToast( - `This draft renames the agent to ${s.path}. Deploy it from the resource editor instead.`, - true - ) - return false - } - // Only a draft naming another type: the load refuses a resource that is not an agent, while a - // draft the generic resource editor wrote names no type at all and inherits the loaded one. - if (s.resource_type && s.resource_type !== 'ai_agent') { - sendUserToast(`This draft is a ${s.resource_type} resource, not an agent.`, true) + const refused = agentDraftDeployRefusal(s, opts.path()) + if (refused) { + sendUserToast(refused, true) return false } // The form stays editable while the request is in flight, so everything below works from a @@ -280,39 +347,15 @@ export function useAgentDraft(opts: AgentDraftOptions): AgentDraftHandle { // made during the request as saved, and the banner would clear on a value the server never // received; against the snapshot it stays a draft, which is what it is. const submitted = structuredClone($state.snapshot(s)) as AgentResourceState - const body = { - path: submitted.path, - value: submitted.args, - description: submitted.description, - labels: submitted.labels, - ws_specific: submitted.wsSpecific - } - try { - if (noDeployed) { - await ResourceService.createResource({ - workspace: ws, - // A create needs a type, and the load proved this path is an agent before opening. - requestBody: { ...body, resource_type: submitted.resource_type ?? 'ai_agent' } - }) + const written = await writeAgentResource(ws, submitted, noDeployed) + if (!written.ok) { + // A path that is no longer an agent tears this editor down; anything else is a plain error + // the user can retry from the form as it stands. + if (written.notAnAgent) { + refuse(written.error) } else { - // The type this editor proved is as old as the load, and an update carries no type of - // its own: were the path deleted and recreated as something else meanwhile, this write - // would put an agent config inside that resource. Reading it again narrows the window - // to the request rather than to however long the editor stayed open. - const current = await ResourceService.getResource({ workspace: ws, path: submitted.path }) - const refused = agentEditorRefusal(submitted.path, current.resource_type) - if (refused) { - refuse(refused) - return false - } - await ResourceService.updateResource({ - workspace: ws, - path: submitted.path, - requestBody: body - }) + sendUserToast(written.error, true) } - } catch (err) { - sendUserToast(`Could not save agent: ${err}`, true) return false } // The counter the step card's write-back used to report, from the surface that now owns the diff --git a/frontend/src/lib/components/flows/agentEditorStore.svelte.ts b/frontend/src/lib/components/flows/agentEditorStore.svelte.ts index 83660212b3..025a426d28 100644 --- a/frontend/src/lib/components/flows/agentEditorStore.svelte.ts +++ b/frontend/src/lib/components/flows/agentEditorStore.svelte.ts @@ -6,6 +6,8 @@ * resources page. Module-level rather than a context value because what opens it — a step's card, * a list row — unmounts the moment the selection moves. */ +import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte' + export interface AgentEditorTarget { path: string /** The workspace the opener operates on; the nav workspace when absent. */ @@ -61,3 +63,25 @@ export function markAgentWritten(workspace: string | undefined, path: string) { export function agentWriteCount(workspace: string | undefined, path: string | undefined): number { return agentWrites[writeKey(workspace, path)] ?? 0 } + +/** How many times each agent's DRAFT has been saved, for the surfaces that display an agent by + * fetching it. A draft write moves no deployed version, so `agentWriteCount` never sees it, and a + * card keyed on that alone would keep describing the config a test no longer runs. */ +let agentDraftSaves = $state>({}) + +/** Every writer in this document goes through the draft syncer — this editor, the generic resource + * editor — so one subscription answers for them all, firing when the write lands rather than on + * each keystroke. `resource` is the item kind agent draft rows use. In-memory: a save in another + * tab never arrives here, so a card lags it until reload, while what a test runs is read live. */ +UserDraftDbSyncer.onAnySaved(({ workspace, itemKind, path }) => { + if (itemKind !== 'resource') return + const key = writeKey(workspace, path) + agentDraftSaves[key] = (agentDraftSaves[key] ?? 0) + 1 +}) + +export function agentDraftSaveCount( + workspace: string | undefined, + path: string | undefined +): number { + return agentDraftSaves[writeKey(workspace, path)] ?? 0 +} diff --git a/frontend/src/lib/components/flows/agentTelemetry.ts b/frontend/src/lib/components/flows/agentTelemetry.ts index fcfad440ff..87fcb0dad4 100644 --- a/frontend/src/lib/components/flows/agentTelemetry.ts +++ b/frontend/src/lib/components/flows/agentTelemetry.ts @@ -13,6 +13,10 @@ export type ReusableAgentEvent = | 'linked' /** A linked step was forked back into a standalone agent. */ | 'unlinked' + /** A linked agent's unsaved draft was deployed alongside the flow that uses it. */ + | 'draft_deployed_with_flow' + /** A linked agent's unsaved draft was left as a draft when its flow was deployed. */ + | 'draft_kept_on_deploy' export function logReusableAgentUsage(event: ReusableAgentEvent): void { logFeatureUsage('ai_agent', 'reusable', { key: event }) diff --git a/frontend/src/lib/components/flows/content/AgentEditorHost.svelte b/frontend/src/lib/components/flows/content/AgentEditorHost.svelte index 79db0cedb3..a7c867a6f4 100644 --- a/frontend/src/lib/components/flows/content/AgentEditorHost.svelte +++ b/frontend/src/lib/components/flows/content/AgentEditorHost.svelte @@ -29,6 +29,7 @@ inputTransformsToAgentConfig, type AIAgentConfig } from '../agentResourceUtils' + import { agentArgsToTransforms } from '../linkedAgentDrafts' import { AGENT_TOOLS_ROW } from '../agentFormFields' import { toolDisplayName, type AgentTool } from '../agentToolUtils' import { useAgentDraft } from '../agentDraft.svelte' @@ -223,19 +224,6 @@ }) }) - /** Every argument the resource carries, as a static transform. `tools` is the roster rather than - * a field, so it rides on the module's own key instead. Not only the keys the form renders: a - * run reads them all, and an agent holding its own `user_message` answers with it when nothing - * overrides it, so a test here has to run the configuration a linked step would. */ - function agentArgsToTransforms(args: AIAgentConfig): Record { - const it: Record = {} - for (const [key, value] of Object.entries(args ?? {})) { - if (key === 'tools' || value === undefined) continue - it[key] = { type: 'static', value } as InputTransform - } - return it - } - /** Everything the form does not model. `inputTransformsToAgentConfig` rebuilds the value from * `AGENT_BRAIN_KEYS` alone, so a key this editor never renders — one a newer backend added, or * the `user_message` default a resource may carry, which the runtime does read when the step diff --git a/frontend/src/lib/components/flows/content/AgentEditorModal.svelte b/frontend/src/lib/components/flows/content/AgentEditorModal.svelte index 3ded2deb9f..22ac424144 100644 --- a/frontend/src/lib/components/flows/content/AgentEditorModal.svelte +++ b/frontend/src/lib/components/flows/content/AgentEditorModal.svelte @@ -183,7 +183,9 @@ const moduleIds = new Set(linkedModulesForAgent(scope, path)) moduleIds.add(at.host.moduleId) return Promise.all( - [...moduleIds].map((moduleId) => publishLinkedAgentTools(path, at.ws, scope, moduleId)) + // With the draft: a deploy leaves none, but a version restore leaves the draft standing and + // it is still what a test of the host step would run. + [...moduleIds].map((moduleId) => publishLinkedAgentTools(path, at.ws, scope, moduleId, true)) ) } diff --git a/frontend/src/lib/components/flows/content/AgentResourceBar.svelte b/frontend/src/lib/components/flows/content/AgentResourceBar.svelte index 3a870f911d..96548b10af 100644 --- a/frontend/src/lib/components/flows/content/AgentResourceBar.svelte +++ b/frontend/src/lib/components/flows/content/AgentResourceBar.svelte @@ -4,7 +4,7 @@ import Badge from '$lib/components/common/badge/Badge.svelte' import Path from '$lib/components/Path.svelte' import TextInput from '$lib/components/text_input/TextInput.svelte' - import { ResourceService, type InputTransform } from '$lib/gen' + import { ResourceService, type InputTransform, type Resource } from '$lib/gen' import { workspaceStore } from '$lib/stores' import { sendUserToast } from '$lib/toast' import { Bot, ChevronDown, ChevronUp, Save, Unlink, Pencil } from 'lucide-svelte' @@ -19,14 +19,24 @@ type AIAgentConfig, type AgentTool } from '../agentResourceUtils' - import { agentWriteCount, markAgentWritten, openAgentEditor } from '../agentEditorStore.svelte' + import { + agentDraftSaveCount, + agentWriteCount, + markAgentWritten, + openAgentEditor + } from '../agentEditorStore.svelte' import { setLinkedAgentTools, clearLinkedAgentTools, + linkedModulesForAgent, linkedToolsScope } from '../linkedAgentToolsStore.svelte' import { logReusableAgentUsage } from '../agentTelemetry' import { claimLinkedToolsFetch } from '../flowState' + import { AgentDraftUnavailable, fetchAgentWithDraft } from '../linkedAgentDrafts' + import type { AgentResourceState } from '../agentDraft.svelte' + import { getLocalDraftHint } from '$lib/localDraftHints.svelte' + import Tooltip from '$lib/components/meltComponents/Tooltip.svelte' import type { AgentTool as AgentToolStrict } from '../agentToolUtils' import { resource } from 'runed' import { untrack } from 'svelte' @@ -65,6 +75,10 @@ // deploy from the agent editor mounted alongside it. Both reads below key on it, so neither // keeps naming the config and version a write has just replaced. let writes = $derived(agentWriteCount(ws, agent)) + // Draft saves as well, for the link fetch: the card shows what a test of this step would run, + // and that is the draft. Only the deploy moves `writes`, so without this the card would keep + // describing the config the agent held before it was edited. + let draftSaves = $derived(agentDraftSaveCount(ws, agent)) let saveDrawer: Drawer | undefined = $state() let newPath = $state('') @@ -75,14 +89,18 @@ type LinkedInfo = { // What this result was fetched for. runed's resource neither aborts nor tags a superseded // request, so a slow fetch can land after a newer one: every consumer gates on these matching - // the current (ws, agent, writes). `writes` is what covers a refetch of the *same* link after - // a deploy — without it a pre-deploy response is indistinguishable from the current one, and - // accepting it republishes the tools the deploy just replaced. + // the current (ws, agent, writes, draftSaves). `writes` is what covers a refetch of the *same* + // link after a deploy — without it a pre-deploy response is indistinguishable from the current + // one, and accepting it republishes the tools the deploy just replaced. `draftSaves` does the + // same for a draft save, which the card follows just as closely. ws?: string path?: string writes: number + draftSaves: number config: AIAgentConfig tools: AgentTool[] + /** The config shown came from the agent's unsaved draft rather than the deployed resource. */ + fromDraft: boolean providerPath?: string providerOk: boolean } @@ -90,14 +108,37 @@ // A linked agent is rigid and read-only: its brain and tools come from the resource. We // load them here for display, and probe the provider resource so we can warn when it isn't // accessible in this workspace (the user then needs to unlink/fork or gain access). + // The draft when there is one, since that is what a test of this step runs. let linkedResource = resource( - () => ({ ws, path: agent, writes }), - async ({ ws, path, writes }): Promise => { + () => ({ ws, path: agent, writes, draftSaves }), + async ({ ws, path, writes, draftSaves }): Promise => { if (!ws || !path) { - return { ws, path, writes, config: {}, tools: [], providerOk: true } + return { + ws, + path, + writes, + draftSaves, + config: {}, + tools: [], + fromDraft: false, + providerOk: true + } + } + let response: Resource + let draft: AgentResourceState | undefined + try { + ;({ response, draft } = await fetchAgentWithDraft(path, ws)) + } catch (err) { + // Only the DRAFT was unreadable. This card is a display, so fall back to the deployed + // agent rather than rendering one with no brain and no tools, which reads as "the agent + // is empty" while the Draft badge still says it has unsaved changes. Same fallback the + // graph's tool nodes take; the paths that run or deploy the draft still refuse. + if (!(err instanceof AgentDraftUnavailable)) throw err + response = await ResourceService.getResource({ workspace: ws, path }) + } + const cfg = (draft?.args ?? response.value ?? {}) as AIAgentConfig & { + provider?: { resource?: string } } - const res = await ResourceService.getResource({ workspace: ws, path }) - const cfg = (res.value ?? {}) as AIAgentConfig & { provider?: { resource?: string } } const tools = (cfg.tools ?? []) as AgentTool[] const providerRef = cfg.provider?.resource const providerPath = @@ -116,8 +157,10 @@ ws, path, writes, + draftSaves, config: cfg, tools, + fromDraft: draft != undefined, providerPath, providerOk } @@ -129,7 +172,13 @@ let loadedInfo = $state(undefined) $effect(() => { const current = linkedResource.current - if (current && current.ws === ws && current.path === agent && current.writes === writes) { + if ( + current && + current.ws === ws && + current.path === agent && + current.writes === writes && + current.draftSaves === draftSaves + ) { loadedInfo = current } }) @@ -140,6 +189,12 @@ let brainParams = $derived(summarizeAgentBrain(linkedInfo?.config)) let providerPath = $derived(linkedInfo?.providerPath) let providerOk = $derived(linkedInfo?.providerOk ?? true) + // The hint flips on the first keystroke in the agent editor, so the badge does not wait for the + // debounced autosave and the refetch behind it; the fetched answer covers a draft written + // elsewhere, which no editor here has published an opinion about. + let hasDraft = $derived( + getLocalDraftHint(ws, 'resource', agent ?? '') ?? linkedInfo?.fromDraft ?? false + ) /** The agent the card is about: the one this step links to, or the one being edited. */ let cardPath = $derived(agent) // The version eval runs are recorded against. The resource does not hold it; its newest history @@ -190,9 +245,18 @@ } const loaded = linkedInfo if (loaded) { - claimLinkedToolsFetch(toolScope, moduleId) - // linkedResource types tools loosely; they are the same resource tools the store holds. - setLinkedAgentTools(toolScope, moduleId, loaded.tools as AgentToolStrict[], agent) + // Every step of this flow linking this agent, not just this one. Tools belong to the agent, + // so the sibling steps show the same set, and only the selected step mounts this card: + // without them a draft saved from here leaves their nodes on what the flow load resolved, + // while a test of those steps runs the draft. Claimed like this card's own publish, so a + // sibling's in-flight fetch cannot land afterwards and put the old tools back. + const modules = new Set(linkedModulesForAgent(toolScope, agent)) + modules.add(moduleId) + for (const id of modules) { + claimLinkedToolsFetch(toolScope, id) + // linkedResource types tools loosely; they are the same resource tools the store holds. + setLinkedAgentTools(toolScope, id, loaded.tools as AgentToolStrict[], agent) + } publishedFor = agent } else if (publishedFor !== undefined && publishedFor !== agent) { // The link moved and the new agent hasn't resolved, so the stored tools are the old one's. @@ -358,13 +422,15 @@ // `tools` is one array per module value, so it identifies the step itself — the path alone // would not, since a replacement can carry the same link. const stepMarker = tools - const res = await ResourceService.getResource({ workspace: ws, path }) + // The draft, like the card above and like a test of this step: forking the deployed value + // while the card displays a drafted prompt would hand back something the user never saw. + const { response, draft } = await fetchAgentWithDraft(path, ws) // The module may have been replaced while the fetch was in flight (undo, session drafts); // applying a stale fork would overwrite the restored state. if (agent !== path || tools !== stepMarker) { return false } - const cfg = (res.value ?? {}) as AIAgentConfig + const cfg = (draft?.args ?? response.value ?? {}) as AIAgentConfig // Preserve the flow-local inputs already wired in the step. const local: Record = {} for (const key of AGENT_FLOW_LOCAL_KEYS) { @@ -448,6 +514,15 @@ v{version} {/if} + {#if hasDraft} + + Draft + {#snippet text()} + This agent has unsaved changes. Testing this flow runs the draft, and deploying the + flow offers to deploy it. + {/snippet} + + {/if}
{#if brainParams.length > 0 || inheritedTools.length > 0} diff --git a/frontend/src/lib/components/flows/flowState.ts b/frontend/src/lib/components/flows/flowState.ts index 3123396a9d..595227f57d 100644 --- a/frontend/src/lib/components/flows/flowState.ts +++ b/frontend/src/lib/components/flows/flowState.ts @@ -5,6 +5,7 @@ import { get } from 'svelte/store' import { workspaceStore } from '$lib/stores' import { isFlowModuleTool, agentToolToFlowModule, type AgentTool } from './agentToolUtils' import { linkedToolsScope, setLinkedAgentTools } from './linkedAgentToolsStore.svelte' +import { fetchAgentWithDraft, normalizeAgentRef } from './linkedAgentDrafts' import { loadFlowModuleState } from './flowStateUtils.svelte' import { emptyFlowModuleState } from './utils.svelte' import type { StateStore } from '$lib/utils' @@ -90,7 +91,9 @@ async function mapFlowModule( // the graph can render its tool nodes. They are display-only (their inputs are edited in // the step panel, which infers schemas itself), so no per-tool module state is loaded — // resource tool ids are not flow-unique and must not key into the flow state. - await publishLinkedAgentTools(agentRef, workspace, scope, flowModule.id) + // Drafts included: every caller of `initFlowState` is a flow editor, where the graph has + // to show the tools a test would run. Read-only viewers publish for themselves. + await publishLinkedAgentTools(agentRef, workspace, scope, flowModule.id, true) } else { // Shape-checked because `tools` is JSON-authored: throwing here would skip the agent's // own state below, leaving it with no schema rather than with no tool schemas. @@ -119,11 +122,17 @@ export async function publishLinkedAgentTools( agentRef: string, workspace: string | undefined, scope: string, - moduleId: string + moduleId: string, + /** Resolve from the agent's unsaved draft when there is one. Editors pass true so the graph + * shows the tool set a test would run; read-only viewers pass false, since a run they are + * displaying used the deployed agent. Required rather than defaulted: an editor call site that + * forgets it republishes the deployed tools over the drafted ones, which reads as the graph + * spontaneously reverting. */ + withDraft: boolean ) { const genKey = `${scope}:${moduleId}` const gen = claimLinkedToolsFetch(scope, moduleId) - const tools = await resolveLinkedAgentTools(agentRef, workspace) + const tools = await resolveLinkedAgentTools(agentRef, workspace, withDraft) if (linkedToolFetchGen.get(genKey) === gen) { setLinkedAgentTools(scope, moduleId, tools, agentRef) } @@ -155,12 +164,27 @@ export function claimLinkedToolsFetch(scope: string, moduleId: string): number { // resource is missing or inaccessible so a broken link never stalls the flow load. export async function resolveLinkedAgentTools( agentRef: string, - workspace?: string + workspace: string | undefined, + withDraft: boolean ): Promise { const ws = workspace ?? get(workspaceStore) if (!ws) return [] - const path = agentRef.replace(/^\$res:/, '').replace(/^res:\/\//, '') + const path = normalizeAgentRef(agentRef) try { + if (withDraft) { + try { + const { response, draft } = await fetchAgentWithDraft(path, ws) + const value = (draft?.args ?? response.value) as { tools?: AgentTool[] } | undefined + return (value?.tools ?? []) as AgentTool[] + } catch { + // The draft read failed for any reason. This is a display, not a run, so fall through to + // the deployed tools rather than showing an agent with none: an empty node list reads as + // "the agent lost its tools" instead of "we could not reach the server". The paths that + // act on a draft — the previews and the deploy dialog — surface the failure instead. + // Not rethrowing anything here: the outer catch turns every throw into `[]`, so a + // rethrow would skip the very fallback this exists for. + } + } const res = await ResourceService.getResource({ workspace: ws, path }) return ((res.value as { tools?: AgentTool[] } | undefined)?.tools ?? []) as AgentTool[] } catch { diff --git a/frontend/src/lib/components/flows/linkedAgentDrafts.test.ts b/frontend/src/lib/components/flows/linkedAgentDrafts.test.ts new file mode 100644 index 0000000000..563007615f --- /dev/null +++ b/frontend/src/lib/components/flows/linkedAgentDrafts.test.ts @@ -0,0 +1,148 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +import { + inlineAgentDraft, + inlineAgentDrafts, + loadLinkedAgentDrafts, + type LinkedAgentDraft +} from './linkedAgentDrafts' +import { ResourceService, type FlowModule, type FlowValue } from '$lib/gen' + +type AiAgentValue = Extract + +function linkedStep(input_transforms: Record): AiAgentValue { + return { + type: 'aiagent', + agent: 'f/team/support', + tool_inputs: { t1: { query: { type: 'javascript', expr: 'flow_input.q' } } }, + input_transforms + } as unknown as AiAgentValue +} + +describe('inlineAgentDraft', () => { + // The overlay order is the worker's: the resource brain first, the step's flow-local inputs on + // top. Reversing it would run the agent author's own `user_message` instead of the flow's. + it('keeps the step wired to the flow while the brain comes from the draft', () => { + const inlined = inlineAgentDraft( + linkedStep({ + user_message: { type: 'javascript', expr: 'flow_input.question' }, + user_attachments: { type: 'static', value: [] } + }), + { + provider: { kind: 'openai', model: 'gpt-4o', resource: '$res:f/team/openai' }, + system_prompt: 'answer in french', + user_message: 'the default the agent carries', + tools: [{ id: 't1', summary: 'search' }] + } as any + ) + + expect(inlined.agent).toBeUndefined() + expect(inlined.tools).toEqual([{ id: 't1', summary: 'search' }]) + expect(inlined.input_transforms).toEqual({ + provider: { + type: 'static', + value: { kind: 'openai', model: 'gpt-4o', resource: '$res:f/team/openai' } + }, + system_prompt: { type: 'static', value: 'answer in french' }, + user_message: { type: 'javascript', expr: 'flow_input.question' }, + user_attachments: { type: 'static', value: [] } + }) + // Host bindings are the step's, not the agent's, and the worker overlays them either way. + expect(inlined.tool_inputs).toEqual({ + t1: { query: { type: 'javascript', expr: 'flow_input.q' } } + }) + }) + + // A linked step carries only the flow-local inputs, but one persisted before linking existed can + // still hold stale brain transforms. They must not shadow the draft the test is meant to run. + it('drops brain transforms the step still carries', () => { + const inlined = inlineAgentDraft( + linkedStep({ + user_message: { type: 'static', value: 'hi' }, + system_prompt: { type: 'static', value: 'stale' } + }), + { system_prompt: 'from the draft' } as any + ) + + expect(inlined.input_transforms).toEqual({ + system_prompt: { type: 'static', value: 'from the draft' }, + user_message: { type: 'static', value: 'hi' } + }) + }) +}) + +describe('inlineAgentDrafts', () => { + // The index is keyed on the bare path while a step may name its agent `$res:`-prefixed, and the + // walk has to reach inside branches and loops. Miss either and every preview silently runs the + // deployed agent — the failure this whole path exists to prevent, and a silent one. + it('reaches a $res:-prefixed link nested in a branch', () => { + const value = { + modules: [ + { + id: 'b', + value: { + type: 'branchone', + default: [], + branches: [ + { + modules: [ + { + id: 'inner', + value: { + type: 'aiagent', + agent: '$res:f/team/support', + tools: [], + input_transforms: { user_message: { type: 'static', value: 'hi' } } + } + } + ] + } + ] + } + } + ] + } as unknown as FlowValue + + const drafts = new Map([ + ['f/team/support', { args: { system_prompt: 'drafted' } } as unknown as LinkedAgentDraft] + ]) + + const inner = (inlineAgentDrafts(value, drafts).modules[0].value as any).branches[0].modules[0] + expect(inner.value.agent).toBeUndefined() + expect(inner.value.input_transforms.system_prompt).toEqual({ + type: 'static', + value: 'drafted' + }) + // The input the flow supplies survives the rewrite. + expect(inner.value.input_transforms.user_message).toEqual({ type: 'static', value: 'hi' }) + }) +}) + +// A link the user cannot resolve is an ordinary state and must not block the flow; anything else is +// an outage, and answering "no draft" to one would silently test or deploy against the deployed +// agent while the editor shows the draft. +describe('loadLinkedAgentDrafts error handling', () => { + function failWith(status: number | undefined) { + return async () => { + const err: Error & { status?: number } = new Error('boom') + err.status = status + throw err + } + } + + beforeEach(() => { + vi.restoreAllMocks() + }) + + it.each([401, 403, 404])('treats %i as no draft', async (status) => { + vi.spyOn(ResourceService, 'getResource').mockImplementation(failWith(status) as any) + await expect(loadLinkedAgentDrafts(['f/team/support'], 'ws')).resolves.toEqual(new Map()) + }) + + it.each([500, undefined])('propagates %s rather than reporting no draft', async (status) => { + vi.spyOn(ResourceService, 'getResource').mockImplementation(failWith(status) as any) + await expect(loadLinkedAgentDrafts(['f/team/support'], 'ws')).rejects.toThrow( + 'Could not load the agent f/team/support' + ) + }) +}) diff --git a/frontend/src/lib/components/flows/linkedAgentDrafts.ts b/frontend/src/lib/components/flows/linkedAgentDrafts.ts new file mode 100644 index 0000000000..8da732e7ff --- /dev/null +++ b/frontend/src/lib/components/flows/linkedAgentDrafts.ts @@ -0,0 +1,229 @@ +import { + ResourceService, + type FlowModule, + type FlowValue, + type InputTransform, + type Resource +} from '$lib/gen' +import { UserDraft } from '$lib/userDraft.svelte' +import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte' +import { canWrite } from '$lib/utils' +import type { UserExt } from '$lib/stores' +import { dfs } from './dfs' +import { flowLocalInputs, type AIAgentConfig } from './agentResourceUtils' +import type { AgentResourceState } from './agentDraft.svelte' +import type { AgentTool } from './agentToolUtils' + +/** A step names its agent bare or as `$res:`/`res://`; all three are the same agent, + * and a draft index has to answer for a lookup written any of those ways. Same normalization as + * `linkedAgentToolsStore`, and as the `trim_start_matches` the worker applies. */ +export function normalizeAgentRef(agentRef: string): string { + return agentRef.replace(/^\$res:/, '').replace(/^res:\/\//, '') +} + +/** Every `ai_agent` resource this flow links to, deduped. `dfs` walks agent tool nodes as well as + * branches and loops, so a nested linked agent tool is included. */ +export function linkedAgentPaths(value: FlowValue | undefined): string[] { + if (!value?.modules) return [] + const paths = new Set() + for (const module of dfs(value.modules, (m) => m)) { + const v = module?.value as { type?: string; agent?: string } | undefined + if (v?.type === 'aiagent' && v.agent) { + paths.add(normalizeAgentRef(v.agent)) + } + } + return [...paths] +} + +/** + * The unsaved draft for an agent, freshest first: the cell an open agent editor is writing, then + * what a `get_draft` response carried. + * + * Only the live cell is reliably current, and only while an editor holds it: `releaseEntry` drops + * the cached write at refcount 0 on purpose, so once the agent editor closes the persisted row is + * the sole answer. Read through `fetchAgentWithDraft`, which settles that row first. + */ +export function agentDraftState( + response: { draft?: unknown }, + path: string, + workspace: string | undefined +): AgentResourceState | undefined { + const live = UserDraft.get('resource', path, { workspace }) + return live ?? (response.draft as AgentResourceState | undefined) +} + +/** A refusal that already names the agent and says what is wrong with it, so a caller wrapping it + * would only repeat itself. */ +export class AgentDraftUnavailable extends Error {} + +/** + * An agent's resource together with the draft a run of it would use. + * + * The flush is what makes the answer current. Autosave is debounced by 1.5s (10s ceiling), and + * closing the agent editor releases the in-memory cell without cancelling that pending POST — so + * testing or deploying right after closing would otherwise read a row the last edits have not + * reached yet. `flush` replays the parked save and is a no-op when there is none. + */ +export async function fetchAgentWithDraft( + path: string, + workspace: string +): Promise<{ response: Resource; draft: AgentResourceState | undefined }> { + const query = { workspace, itemKind: 'resource' as const, path } + await UserDraftDbSyncer.flush(query) + // `flush` resolves whether or not the save actually landed: `postSave` catches network and + // server errors into its failure map, and answers a conflicting write by parking a snapshot, + // returning normally in both cases. The row about to be read is then older than the edit still + // held in the browser, and nothing downstream could tell. Running that row is a test of the + // wrong agent; deploying it is worse, because the deploy deletes the draft and takes the newer + // edit with it. Neither is recoverable from here, so refuse the read. + const failure = UserDraftDbSyncer.getState(query).failureMessage + if (failure) { + throw new AgentDraftUnavailable(`The unsaved changes to ${path} could not be saved: ${failure}`) + } + if (UserDraftDbSyncer.getConflict(query).conflict) { + throw new AgentDraftUnavailable( + `The unsaved changes to ${path} could not be saved because it was edited elsewhere. Open the agent to resolve it.` + ) + } + const response = await ResourceService.getResource({ workspace, path, getDraft: true }) + return { response, draft: agentDraftState(response, path, workspace) } +} + +/** One linked agent whose resource the user has an unsaved draft for. */ +export interface LinkedAgentDraft { + path: string + /** The draft's resource value: what a run of this agent would use. */ + args: AIAgentConfig + /** The whole draft row, as the resource editors write it — the deploy payload. */ + state: AgentResourceState + /** No deployed row at this path, so deploying has to create rather than update. */ + noDeployed: boolean + /** Of the deployed resource, for `agentDraftCanWrite`. */ + extraPerms: Record +} + +/** Whether `user` may write this agent's resource. Split from the load so that resolving the + * drafts of a whole flow costs no `whoami` — only the deploy dialog needs the answer, and it + * looks the user up once for every agent it lists. */ +export function agentDraftCanWrite(draft: LinkedAgentDraft, user: UserExt | undefined): boolean { + return canWrite(draft.path, draft.extraPerms, user) +} + +/** A link that cannot resolve for the user rather than because something went wrong: the agent was + * deleted, or sits in a folder they cannot read. Both are ordinary states of a rigid link, and + * neither should stop the caller — the flow still tests and deploys, against the deployed agent. + * Every other failure is an outage, and answering "no draft" to one would quietly run or deploy + * the wrong configuration, which is the whole thing this module exists to prevent. */ +function isExpectedLinkFailure(err: unknown): boolean { + const status = (err as { status?: number } | null | undefined)?.status + return status === 401 || status === 403 || status === 404 +} + +/** + * The unsaved draft of every given `ai_agent` path, for the paths that have one. + * + * Throws when a path fails to load for any reason other than being missing or unreadable, so a + * caller cannot mistake an outage for an agent with nothing unsaved. + */ +export async function loadLinkedAgentDrafts( + paths: string[], + workspace: string | undefined +): Promise> { + const out = new Map() + if (!workspace || paths.length === 0) return out + await Promise.all( + paths.map(async (path) => { + let response: Resource + let draft: AgentResourceState | undefined + try { + ;({ response, draft } = await fetchAgentWithDraft(path, workspace)) + } catch (err) { + if (isExpectedLinkFailure(err)) return + if (err instanceof AgentDraftUnavailable) throw err + throw new Error(`Could not load the agent ${path}: ${err}`) + } + if (!draft) return + out.set(path, { + path, + args: (draft.args ?? {}) as AIAgentConfig, + state: draft, + noDeployed: Boolean((response as { no_deployed?: boolean }).no_deployed), + extraPerms: response.extra_perms ?? {} + }) + }) + ) + return out +} + +/** + * Every argument a saved agent carries, as a static input transform. Not only the keys the agent + * form renders: a run reads them all, and an agent holding its own `user_message` answers with it + * when nothing overrides it. `tools` is the step's own roster rather than an input, so it rides on + * the module's `tools` key instead. + */ +export function agentArgsToTransforms(args: AIAgentConfig): Record { + const it: Record = {} + for (const [key, value] of Object.entries(args ?? {})) { + if (key === 'tools' || value === undefined) continue + it[key] = { type: 'static', value } as InputTransform + } + return it +} + +type AiAgentValue = Extract + +/** + * The standalone step a linked step's draft would run as: the draft's brain and tools inlined, with + * the step's own flow-local inputs kept on top. + * + * The overlay order is the worker's (`ai_executor.rs`): its linked branch interpolates the whole + * resource brain and only then writes `user_message`/`user_attachments` back from the step's own + * args. `tool_inputs` stays untouched — the worker overlays it onto the tools in both branches, so + * an inlined step keeps the host flow's tool bindings. + */ +export function inlineAgentDraft(value: AiAgentValue, args: AIAgentConfig): AiAgentValue { + const { agent: _agent, ...rest } = value + return { + ...rest, + tools: (args.tools ?? []) as AgentTool[], + input_transforms: { + ...agentArgsToTransforms(args), + ...flowLocalInputs(value.input_transforms as Record) + } + } as AiAgentValue +} + +/** + * Replace every linked agent step that has a draft with the draft's own configuration, so a preview + * runs what the agent editor is showing rather than the deployed resource. Returns a new value: the + * flow editor hands its live store object to previews. + */ +export function inlineAgentDrafts( + value: FlowValue, + drafts: Map +): FlowValue { + if (drafts.size === 0) return value + // JSON rather than `structuredClone`: the flow editor's value is a Svelte `$state` proxy, which + // `structuredClone` refuses outright. A flow value is JSON by definition — it is about to be + // posted as one — so the round trip loses nothing this preview would have carried. + const next = JSON.parse(JSON.stringify(value)) as FlowValue + for (const module of dfs(next.modules ?? [], (m) => m)) { + const v = module?.value as AiAgentValue | undefined + if (v?.type !== 'aiagent' || !v.agent) continue + const draft = drafts.get(normalizeAgentRef(v.agent)) + if (!draft) continue + module.value = inlineAgentDraft(v, draft.args) + } + return next +} + +/** Load the drafts this flow's linked agents have and inline them. The whole substitution, for a + * caller holding nothing but the value it is about to preview. */ +export async function withAgentDrafts( + value: FlowValue, + workspace: string | undefined +): Promise { + const paths = linkedAgentPaths(value) + if (paths.length === 0) return value + return inlineAgentDrafts(value, await loadLinkedAgentDrafts(paths, workspace)) +} diff --git a/frontend/src/lib/components/flows/linkedToolsFetchGuard.test.ts b/frontend/src/lib/components/flows/linkedToolsFetchGuard.test.ts index ea1674e0b6..29207f58f7 100644 --- a/frontend/src/lib/components/flows/linkedToolsFetchGuard.test.ts +++ b/frontend/src/lib/components/flows/linkedToolsFetchGuard.test.ts @@ -8,7 +8,9 @@ vi.mock('./agentToolUtils', () => ({ isFlowModuleTool: () => false, agentToolToFlowModule: (t: unknown) => t })) -vi.mock('$lib/stores', () => ({ workspaceStore: { subscribe: (f: (v: string) => void) => (f('ws'), () => {}) } })) +vi.mock('$lib/stores', () => ({ + workspaceStore: { subscribe: (f: (v: string) => void) => (f('ws'), () => {}) } +})) import { claimLinkedToolsFetch, @@ -38,8 +40,8 @@ describe('linked tools fetch guard', () => { ) .mockResolvedValueOnce({ value: { tools: [tool('new')] } } as never) - const stale = publishLinkedAgentTools('f/a/old', 'ws', scope, 'step') - await publishLinkedAgentTools('f/a/new', 'ws', scope, 'step') + const stale = publishLinkedAgentTools('f/a/old', 'ws', scope, 'step', false) + await publishLinkedAgentTools('f/a/new', 'ws', scope, 'step', false) release?.({ value: { tools: [tool('old')] } }) await stale @@ -56,7 +58,7 @@ describe('linked tools fetch guard', () => { () => new Promise((r) => (release = r)) as ReturnType ) - const inFlight = publishLinkedAgentTools('f/a/old', 'ws', scope, 'step') + const inFlight = publishLinkedAgentTools('f/a/old', 'ws', scope, 'step', false) setLinkedAgentTools(scope, 'step', [tool('kept')], 'u/admin/a') invalidateLinkedToolsFetches(scope) @@ -73,7 +75,7 @@ describe('linked tools fetch guard', () => { () => new Promise((r) => (release = r)) as ReturnType ) - const inFlight = publishLinkedAgentTools('f/a/old', 'ws', scope, 'step') + const inFlight = publishLinkedAgentTools('f/a/old', 'ws', scope, 'step', false) claimLinkedToolsFetch(scope, 'step') setLinkedAgentTools(scope, 'step', [tool('direct')], 'u/admin/a') release?.({ value: { tools: [tool('stale')] } }) diff --git a/frontend/src/lib/components/flows/utils.svelte.ts b/frontend/src/lib/components/flows/utils.svelte.ts index 8d8e4366e5..df27960dc7 100644 --- a/frontend/src/lib/components/flows/utils.svelte.ts +++ b/frontend/src/lib/components/flows/utils.svelte.ts @@ -17,6 +17,7 @@ import { get } from 'svelte/store' import type { FlowModuleState } from './flowState' import { type PickableProperties, dfs } from './previousResults' import { forEachFlowModule } from './dfs' +import { withAgentDrafts } from './linkedAgentDrafts' import { NEVER_TESTED_THIS_FAR } from './models' import { sendUserToast } from '$lib/toast' import type { ExtendedOpenFlow } from './types' @@ -187,6 +188,12 @@ export function jobsToResults(jobs: Job[]) { }) } +/** + * Run the flow the editor currently holds. A step linked to a saved agent runs that agent's + * unsaved draft when there is one (`withAgentDrafts`), so testing exercises what the agent editor + * is showing rather than the deployed resource — the same rule the agent editor's own test pane + * follows. The value passed in is left alone; only what goes to the server is substituted. + */ export async function runFlowPreview( args: Record, flow: OpenFlow & { tag?: string }, @@ -198,14 +205,15 @@ export async function runFlowPreview( // editor; falls back to the navigation workspace for full-page previews. workspace?: string ) { - const newFlow = flow + const ws = workspace ?? get(workspaceStore) ?? '' + const value = await withAgentDrafts(flow.value, ws) return await JobService.runFlowPreview({ - workspace: workspace ?? get(workspaceStore) ?? '', + workspace: ws, requestBody: { args, - value: newFlow.value, + value, path: path, - tag: newFlow.tag, + tag: flow.tag, restarted_from: restartedFrom, temp_script_refs: tempScriptRefs }, diff --git a/frontend/src/lib/userDraft.svelte.ts b/frontend/src/lib/userDraft.svelte.ts index 24572ee1ab..7942694f31 100644 --- a/frontend/src/lib/userDraft.svelte.ts +++ b/frontend/src/lib/userDraft.svelte.ts @@ -349,6 +349,32 @@ export const UserDraft = { void UserDraftDbSyncer.save({ workspace: ws, itemKind, path, value: null }) }, + /** + * Drop this key's in-memory cell and cached write WITHOUT touching the + * server. For callers that have already deleted the row by another route + * (a deploy) and only need the local mirror to stop answering `get`/`has` + * with a value that is gone. + * + * MUST be used instead of `remove` there. `remove` POSTs its own + * `value: null`, and that POST is debounced and carries whatever + * `last_sync` is left — which a preceding successful delete has already + * cleared. The backend treats a delete with no `last_sync` as + * unconditional, so the second POST lands ~1.5s later with nothing to + * compare against and removes a draft saved in the meantime. + */ + forgetLocal(itemKind: UserDraftItemKind, path: string, opts?: UserDraftOptions): void { + const ws = resolveWorkspace(opts) + const mk = mapKey(ws, itemKind, path) + const entry = entries.get(mk) + if (entry) { + // Same as `remove`: clear the cell so live observers see the delete, and arm + // `skipNextSync` so the mirror does not turn that write into a POST of its own. + entry.skipNextSync = true + entry.state.val = undefined + } + writtenCache.delete(mk) + }, + clear(itemKind: UserDraftItemKind, path: string, opts?: UserDraftOptions): void { UserDraft.discard(itemKind, path, undefined, opts) }, diff --git a/frontend/src/lib/utils_draft_deploy.test.ts b/frontend/src/lib/utils_draft_deploy.test.ts index 9bed95917b..4495155f24 100644 --- a/frontend/src/lib/utils_draft_deploy.test.ts +++ b/frontend/src/lib/utils_draft_deploy.test.ts @@ -7,7 +7,7 @@ vi.mock('$lib/gen', () => ({ DraftService: { deleteDraft: vi.fn() }, AppService: {}, VariableService: {}, - ResourceService: {}, + ResourceService: { getResource: vi.fn(), updateResource: vi.fn(), createResource: vi.fn() }, ScheduleService: {}, HttpTriggerService: {}, WebsocketTriggerService: {}, @@ -21,7 +21,9 @@ vi.mock('$lib/gen', () => ({ AzureTriggerService: {}, EmailTriggerService: {} })) -vi.mock('$lib/userDraftDbSyncer.svelte', () => ({ UserDraftDbSyncer: { save: vi.fn() } })) +vi.mock('$lib/userDraftDbSyncer.svelte', () => ({ + UserDraftDbSyncer: { save: vi.fn(), recordRemoteSync: vi.fn() } +})) vi.mock('$lib/workspaceDrafts.svelte', () => ({ invalidateWorkspaceDrafts: vi.fn() })) vi.mock('$lib/workspaceComparison', () => ({ invalidateWorkspaceComparison: vi.fn() })) vi.mock('$lib/localDraftHints.svelte', () => ({ setLocalDraftHint: vi.fn() })) @@ -30,7 +32,8 @@ vi.mock('$lib/components/raw_apps/utils', () => ({ canonicalRawAppDiffValue: vi. vi.mock('$lib/appDiffSides', () => ({ classicAppDraftParts: vi.fn() })) vi.mock('$lib/utils_deployable', () => ({ TRIGGER_RUNTIME_IGNORE: [] })) -import { ScriptService, FlowService } from '$lib/gen' +import { ScriptService, FlowService, ResourceService } from '$lib/gen' +import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte' // draftBaseIsStale compares a draft's base pointer against the deployed head // of the item it was fetched with (`get_draft=true`). Shared by CompareDrafts @@ -124,3 +127,65 @@ describe('deployDraft preserves on_behalf_of', () => { ) }) }) + +// The resource branch reads the item again when the deploy lands, and falls back to the deployed +// row when the draft has gone. That row keeps its value under `value` and carries no `args` at +// all, so reading it as a draft (`value: d.args ?? {}`) would replace a live resource with `{}`. +describe('deployDraft: resource with no draft', () => { + beforeEach(() => vi.clearAllMocks()) + + it('writes nothing rather than `{}` over the deployed value', async () => { + vi.mocked(ResourceService.getResource).mockResolvedValueOnce({ + path: 'f/support/triage_agent', + resource_type: 'ai_agent', + value: { system_prompt: 'deployed' } + } as any) + + // `noop` is what lets a caller deploying one specific draft tell "nothing to promote" apart + // from "deployed", instead of reporting an agent as deployed that was never written. + expect(await deployDraft('resource', 'f/support/triage_agent', 'ws')).toEqual({ + success: true, + noop: true + }) + expect(ResourceService.updateResource).not.toHaveBeenCalled() + expect(ResourceService.createResource).not.toHaveBeenCalled() + // Nor does it touch the draft row. There was none of this user's to delete, so the only row + // the cleanup could reach is one written after the read: an edit destroyed without ever + // having been deployed. Clearing the baseline would be the same bug by another route, since + // a delete with no baseline is the unconditional one. + expect(UserDraftDbSyncer.save).not.toHaveBeenCalled() + expect(UserDraftDbSyncer.recordRemoteSync).not.toHaveBeenCalled() + }) + + it('still deploys normally when the draft is there, and keys the cleanup to the row it read', async () => { + vi.mocked(ResourceService.getResource).mockResolvedValueOnce({ + path: 'f/support/triage_agent', + resource_type: 'ai_agent', + value: { system_prompt: 'deployed' }, + draft_saved_at: '2026-01-01T00:00:00Z', + draft: { path: 'f/support/triage_agent', args: { system_prompt: 'drafted' } } + } as any) + + expect(await deployDraft('resource', 'f/support/triage_agent', 'ws')).toEqual({ success: true }) + expect(ResourceService.updateResource).toHaveBeenCalledWith( + expect.objectContaining({ + requestBody: expect.objectContaining({ value: { system_prompt: 'drafted' } }) + }) + ) + // The draft delete that follows is conditional on this baseline. With no baseline the backend + // deletes unconditionally, destroying a draft saved between the read and the delete without + // ever having deployed it, so the timestamp has to be the one from the row just promoted. + expect(UserDraftDbSyncer.recordRemoteSync).toHaveBeenCalledWith( + { workspace: 'ws', itemKind: 'resource', path: 'f/support/triage_agent' }, + '2026-01-01T00:00:00Z' + ) + expect(UserDraftDbSyncer.save).toHaveBeenCalledWith( + expect.objectContaining({ path: 'f/support/triage_agent', value: null, immediate: true }) + ) + // Order is the whole point: a delete issued before the seed carries whatever baseline the tab + // happened to hold, which for a caller that only read through a listing is none at all. + expect(vi.mocked(UserDraftDbSyncer.recordRemoteSync).mock.invocationCallOrder[0]).toBeLessThan( + vi.mocked(UserDraftDbSyncer.save).mock.invocationCallOrder[0] + ) + }) +}) diff --git a/frontend/src/lib/utils_draft_deploy.ts b/frontend/src/lib/utils_draft_deploy.ts index b322d6adc4..f322bb27b9 100644 --- a/frontend/src/lib/utils_draft_deploy.ts +++ b/frontend/src/lib/utils_draft_deploy.ts @@ -435,8 +435,13 @@ export async function deployDraft( path: string, workspace: string, opts: { draftOnly?: boolean; rawApp?: boolean; deploymentMessage?: string } = {} -): Promise { +): Promise { const { draftOnly = false, rawApp = false, deploymentMessage } = opts + // Set when the branch found nothing to promote and wrote nothing. Success, because the item is + // already at the value a deploy would have left it at and its stale draft state still wants + // clearing — but a caller deploying one specific draft it showed the user has to be able to tell + // that apart from having deployed it. + let noop = false try { if (kind === 'raw_app' || (kind === 'app' && rawApp)) { // Raw apps bundle their source files and deploy via the raw-app @@ -579,10 +584,34 @@ export async function deployDraft( } void deployed } else if (kind === 'resource') { - const { deployed, draft: d } = splitOverlay(await OVERLAY_GETTERS.resource!(workspace, path)) + const overlay = await OVERLAY_GETTERS.resource!(workspace, path) + // Adopt the row this promote is based on as the baseline for the delete below. Without one + // the backend deletes unconditionally, so a draft saved between that read and the delete is + // destroyed having never been deployed — a caller that only ever read through a listing has + // no baseline of its own to supply. With it the delete is refused instead and the newer + // draft survives, which is the recoverable outcome of the two. Only ever seeded, never + // cleared: passing no timestamp drops whatever baseline the tab already held, which would + // turn that same delete back into an unconditional one. + if (overlay?.draft_saved_at) { + UserDraftDbSyncer.recordRemoteSync( + { workspace, itemKind: kind, path }, + overlay.draft_saved_at + ) + } + const { deployed, draft: d, hasDraft } = splitOverlay(overlay) // ResourceEditor's `ResourceState` draft shape: // { path, description, args, resource_type?, labels?, wsSpecific } - if (draftOnly) { + // The deployed row is a different shape (`value`, `ws_specific`, no `args` at all), and + // `splitOverlay` hands it back as the draft side when the draft row has gone — deployed or + // discarded from another tab between the listing and this click. Reading it as a draft is + // what made `value: d.args ?? {}` replace a live resource with `{}`. Nothing to promote + // then, so write nothing and fall through to the cleanup below, which clears the stale + // local draft hint and the drafts listing. The item is already at the value a successful + // deploy would have left it at, so this reports success rather than an error, matching + // what the other kinds end up doing when their own draft is gone. + if (!hasDraft) { + noop = true + } else if (draftOnly) { await ResourceService.createResource({ workspace, requestBody: { @@ -633,8 +662,8 @@ export async function deployDraft( return { success: false, error: `Deploy not supported for draft kind ${kind}` } } // Delete the draft at its STORAGE path (the row key, = the `path` arg). - // Two reasons it must happen here for every kind, mirroring the editors' - // post-deploy `discardDraftAfterDeploy(draftPath)`: + // Two reasons it must happen here for every kind that promoted something, + // mirroring the editors' post-deploy `discardDraftAfterDeploy(draftPath)`: // - Drawer kinds (variable / resource / triggers) aren't deleted by // their create/update endpoints at all. // - script/flow/app/raw_app DO delete server-side, but only the draft at @@ -642,13 +671,18 @@ export async function deployDraft( // synthetic `u/{user}/draft_{uuid}` storage path ≠ `d.path`, so its // draft row survives the deploy and keeps listing. Deleting the // storage-path draft removes it (a no-op when the server already did). - await UserDraftDbSyncer.save({ - workspace, - itemKind: kind, - path, - value: null, - immediate: true - }) + // Skipped when nothing was promoted: the read that set `noop` found no draft of this user's to + // delete, so the only row this could reach is one written after it — destroying an edit that + // was never deployed, and never even listed. + if (!noop) { + await UserDraftDbSyncer.save({ + workspace, + itemKind: kind, + path, + value: null, + immediate: true + }) + } // Mutated the workspace's Server Drafts — refresh every mounted reader. invalidateWorkspaceDrafts(workspace) // The DEPLOYED state moved: cached fork comparisons involving this @@ -659,7 +693,7 @@ export async function deployDraft( // so the syncer-owned hint won't auto-clear — clear it explicitly. // (Idempotent: the drawer-kind delete above already cleared it.) setLocalDraftHint(workspace, kind, path, false) - return { success: true } + return noop ? { success: true, noop: true } : { success: true } } catch (e: any) { return { success: false, error: e?.body ?? e?.message ?? String(e) } } From c3f7f8a45830fb548aa628ebf6e2b6c95c6de67f Mon Sep 17 00:00:00 2001 From: Diego Imbert <70353967+diegoimbert@users.noreply.github.com> Date: Mon, 7 Sep 2026 19:31:59 +0200 Subject: [PATCH 014/144] fix: stop an untouched item's form from saving a draft nobody wrote (#10964) * feat: gate drafts on real user input so a moved-on schema is not a draft Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01ScVgqGpuyDMWdzVNPm7Q5f * Revert "feat: gate drafts on real user input so a moved-on schema is not a draft" This reverts commit 6cd86cf727dc46d9836440ae345ca6161c3fb31c. * fix: stop counting empty schema-added fields and server metadata as drafts Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01ScVgqGpuyDMWdzVNPm7Q5f * feat: sweep away existing drafts that carry no changes, once per workspace Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01ScVgqGpuyDMWdzVNPm7Q5f * Reapply "feat: gate drafts on real user input so a moved-on schema is not a draft" This reverts commit b7b18e345e6984544290009247faca1e341b46bd. * Revert "fix: stop counting empty schema-added fields and server metadata as drafts" This reverts commit 9787270ad883722735d3873ee98d37a7fd11ad23. * docs: describe the sweep by the gate that now prevents new phantom drafts Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01ScVgqGpuyDMWdzVNPm7Q5f * fix: make the draft sweep a compare-and-delete so it cannot eat live edits Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01ScVgqGpuyDMWdzVNPm7Q5f * fix: close the gate's load-time window and stop sealing a failed sweep Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01ScVgqGpuyDMWdzVNPm7Q5f * fix: open the gate on the edit itself, and stop the sweep at ownerless drafts Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01ScVgqGpuyDMWdzVNPm7Q5f * fix: count a click as an edit, and keep an unjudged row from sealing the sweep Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01ScVgqGpuyDMWdzVNPm7Q5f * fix: release the sweep's sync baseline when its delete is refused Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01ScVgqGpuyDMWdzVNPm7Q5f * fix: drop the refused delete before re-baselining, and bound the sweep's retries Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01ScVgqGpuyDMWdzVNPm7Q5f * refactor: send the sweep's delete straight to the API, not through the syncer Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01ScVgqGpuyDMWdzVNPm7Q5f * fix: never absorb a change the resource type's schema could not have made Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01ScVgqGpuyDMWdzVNPm7Q5f * fix: push an edit the gate only notices after the write has landed Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01ScVgqGpuyDMWdzVNPm7Q5f * fix: stop the gating effect re-suspending a resource opened on a draft Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01ScVgqGpuyDMWdzVNPm7Q5f * chore: update ee-repo-ref to d33ea730c550cdbc7d050aeb6d40dcef3d134e07 This commit updates the EE repository reference after PR #782 was merged in windmill-ee-private. Previous ee-repo-ref: 313c572c9dcbcaafd8a1594df4054f9dd26f395c New ee-repo-ref: d33ea730c550cdbc7d050aeb6d40dcef3d134e07 Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: windmill-internal-app[bot] --- .../src/lib/components/ResourceEditor.svelte | 108 +++++++- .../triggers/useTriggerDraftSync.svelte.ts | 106 ++++++-- frontend/src/lib/userDraft.svelte.ts | 5 +- frontend/src/lib/userDraftEditGate.ts | 66 +++++ frontend/src/lib/userDraftPrune.test.ts | 243 ++++++++++++++++++ frontend/src/lib/userDraftPrune.ts | 230 +++++++++++++++++ frontend/src/lib/utils_draft_deploy.ts | 13 + .../src/routes/(root)/(logged)/+layout.svelte | 11 +- 8 files changed, 759 insertions(+), 23 deletions(-) create mode 100644 frontend/src/lib/userDraftEditGate.ts create mode 100644 frontend/src/lib/userDraftPrune.test.ts create mode 100644 frontend/src/lib/userDraftPrune.ts diff --git a/frontend/src/lib/components/ResourceEditor.svelte b/frontend/src/lib/components/ResourceEditor.svelte index 89acac79de..3c9eda9467 100644 --- a/frontend/src/lib/components/ResourceEditor.svelte +++ b/frontend/src/lib/components/ResourceEditor.svelte @@ -2,7 +2,7 @@ import type { Schema } from '$lib/common' import { ResourceService, WorkspaceService, type Resource, type ResourceType } from '$lib/gen' import { canWrite } from '$lib/utils' - import { createEventDispatcher, untrack } from 'svelte' + import { createEventDispatcher, onDestroy, untrack } from 'svelte' import { userStore, workspaceStore } from '$lib/stores' import { sendUserToast } from '$lib/toast' import { clearJsonSchemaResourceCache } from './schema/jsonSchemaResource.svelte' @@ -13,7 +13,9 @@ import { getUserExt } from '$lib/user' import type { UserExt } from '$lib/stores' import { UserDraft, draftValuesEqual, type UserDraftHandle } from '$lib/userDraft.svelte' + import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte' import { setLocalDraftHint } from '$lib/localDraftHints.svelte' + import { onUserInput } from '$lib/userDraftEditGate' interface Props { canSave?: boolean @@ -108,6 +110,55 @@ workspaceSpecs.push({ ws, defaultValue }) } + // Gated per workspace until the user puts something into that workspace's + // form (see `onUserInput`): the autosave stays suspended and the deployed + // baseline absorbs whatever the form settles on. A workspace opened ON a + // saved draft keeps its baseline — that divergence is the user's own. + let userEdited: Record = $state({}) + let openedOnDraft: Record = $state({}) + const suspendedWorkspaces = new Set() + + function setGated(ws: string, gated: boolean): void { + if (!initialPath) return + if (gated === suspendedWorkspaces.has(ws)) return + if (gated) { + UserDraft.stopSync('resource', initialPath, { workspace: ws }) + suspendedWorkspaces.add(ws) + } else { + UserDraft.restartSync('resource', initialPath, { workspace: ws }) + suspendedWorkspaces.delete(ws) + } + } + + // Nothing counts until this workspace's form is on screen, and while the + // schema is still arriving a precursor alone does not: it would open the gate + // just in time for the schema's materialized values to POST. `Path`, the + // labels and the description render above that skeleton and stay editable + // throughout, so a real value event still counts and keeps the edit. + onUserInput((kind) => { + if (!selected || !(selected in states)) return + if (kind === 'precursor' && loadingSchema) return + userEdited[selected] = true + }) + + $effect(() => { + const wss = Object.keys(states) + const edited = { ...userEdited } + const onDraft = { ...openedOnDraft } + untrack(() => { + // A workspace opened on a saved draft is never suspended — there is no + // phantom to prevent, and a write made while suspended is dropped for + // good. Without `onDraft` here this effect re-suspends it the moment its + // handle appears, undoing the decision made when it was opened. + for (const ws of wss) setGated(ws, !edited[ws] && !onDraft[ws]) + }) + }) + + // `stopSync` must be paired or the key stays unsynced for the session. + onDestroy(() => { + for (const ws of [...suspendedWorkspaces]) setGated(ws, false) + }) + let isValid = $state(true) let jsonError = $state('') let perWsValid: Record = $state({}) @@ -245,6 +296,13 @@ } // Open with the saved draft if present, else the deployed. const s: ResourceState = savedDraftState ?? deployedState + openedOnDraft[ws] = !!savedDraftState + // Gate BEFORE the handle is acquired: `stopSync` queues on a + // not-yet-live entry, and the form can settle before the effect + // above gets a chance to run. Only worth doing when no draft exists + // yet — where one does, there is no phantom to prevent and + // suspending could only drop a write. + if (!savedDraftState) setGated(ws, true) ensureHandle(ws, s) initialStates[ws] = structuredClone(deployedState) // Draft-only paths (`no_deployed`) have no row — saving must @@ -259,6 +317,47 @@ }) }) + /** The schema can only ever write `args`. `path`, `labels`, `description` and + * `wsSpecific` are beyond its reach, so a difference in one of those is the + * user's — whatever event did or didn't reach the gate. Removing a label runs + * a click handler and emits nothing native, and would otherwise be absorbed. */ + function differsOutsideArgs(a: ResourceState, b: ResourceState | undefined): boolean { + return !!b && !draftValuesEqual({ ...a, args: null }, { ...b, args: null }) + } + + // Absorb the form's settling writes into the deployed baseline while the + // selected workspace is gated, so they show up neither as the "unsaved + // changes" banner nor, once `discardIf` reads the baseline, as a draft. + // Only the selected workspace has a form rendered against it. + $effect(() => { + const ws = selected + if (!ws || !initialPath) return + if (userEdited[ws] || openedOnDraft[ws]) return + // `$state.snapshot` deep-reads, so nested `args` mutations re-run this. + const settled = states[ws]?.draft + ? ($state.snapshot(states[ws].draft) as ResourceState) + : undefined + untrack(() => { + if (!settled) return + if (differsOutsideArgs(settled, initialStates[ws])) { + // An edit, not settling. This runs AFTER the write landed, and a write + // made while suspended is swallowed for good (the mirror advances its + // baseline either way), so un-suspend and push the value here rather + // than leaving it to whichever effect happens to run next. + userEdited[ws] = true + setGated(ws, false) + void UserDraftDbSyncer.save({ + workspace: ws, + itemKind: 'resource', + path: initialPath, + value: settled + }) + return + } + if (!draftValuesEqual(settled, initialStates[ws])) initialStates[ws] = settled + }) + }) + // Keep current.path bound to the outer `path` prop for consumers $effect(() => { if (current) path = current.path @@ -292,6 +391,13 @@ } export function discardLocalDraft(): void { if (!selected) return + // Back to the deployed value with nothing of the user's left in it, so + // the gate closes again — otherwise the form settles on the schema's + // values a second time and the discarded draft comes straight back. + // `discard` POSTs the delete itself, so suspending first is safe. + openedOnDraft[selected] = false + userEdited[selected] = false + setGated(selected, true) UserDraft.discard('resource', initialPath ?? '', initialStates[selected], { workspace: selected }) diff --git a/frontend/src/lib/components/triggers/useTriggerDraftSync.svelte.ts b/frontend/src/lib/components/triggers/useTriggerDraftSync.svelte.ts index ba09167d6d..98d38ac25f 100644 --- a/frontend/src/lib/components/triggers/useTriggerDraftSync.svelte.ts +++ b/frontend/src/lib/components/triggers/useTriggerDraftSync.svelte.ts @@ -2,9 +2,21 @@ import { untrack } from 'svelte' import { deepEqual } from 'fast-equals' import { UserDraft, normalizeDraftForCompare, type UserDraftItemKind } from '$lib/userDraft.svelte' import { setLocalDraftHint } from '$lib/localDraftHints.svelte' +import { onUserInput } from '$lib/userDraftEditGate' type Cfg = Record +/** + * Detach a config from whatever holds it. The draft cell is deeply reactive, + * so handing its object straight to `applyCfg` would make the form's own + * `$state` (a schedule's `args`, say) the very object the cell holds — every + * later keystroke would then mutate the draft in place behind the autosave's + * back. + */ +function snapshotCfg(cfg: V): V { + return structuredClone($state.snapshot(cfg)) as V +} + /** * Whether `a` differs from `b` after `normalizeDraftForCompare` (JSON * round-trip to drop `undefined`-valued keys, plus ignored deploy-directive @@ -84,8 +96,10 @@ export interface TriggerDraftSync { * …)` (another tab, a programmatic write) propagate into the open editor. * * - **apply-effect**: reflects external `handle.draft` changes into the form. + * - **absorb-effect**: folds the form's own settling into the baseline until + * the user's first input, so a schema that moved on is not a draft. * - **persist-effect**: writes form edits back through the handle, dropping - * the draft when the form is back at the deployed baseline. + * the draft when the form is back at the baseline. * * Both effect bodies are `untrack`ed and gated by `cfgDiffers` * idempotence so they can't feed back into each other. Must be called once @@ -99,14 +113,56 @@ export function useTriggerDraftSync(opts: TriggerDraftSyncOptions): TriggerDraft }) const handle = $derived(handles[0]) + // Gated until the user puts something in (see `onUserInput`): the baseline + // absorbs whatever the form settles on and nothing persists, so an untouched + // trigger never reports unsaved changes. A drawer opened ON a restored draft + // absorbs nothing — that divergence is the user's own. + let settledBaseline: Cfg | undefined = $state(undefined) + let userEdited = $state(false) + let openedOnDraft = $state(false) + // These editors are mounted by the list page, not by the drawer, so input + // arriving while the drawer is still loading is the click that opened it + // (or anything else on the page behind) — never an edit to this form. + onUserInput(() => { + if (!opts.drawerLoading()) userEdited = true + }) + + /** The deployed config, plus whatever the form settled on by itself. */ + const baseline = $derived(settledBaseline ?? opts.deployed()) + + $effect(() => { + // A reload re-opens the gate's window: the drawer is being pointed at a + // different trigger, or the same one re-read from the backend. The click + // that opened it landed before this, hence the reset of `userEdited`. + if (!opts.drawerLoading()) return + untrack(() => { + settledBaseline = undefined + userEdited = false + openedOnDraft = false + }) + }) + + // absorb-effect: pre-edit form drift joins the baseline. + $effect(() => { + if (opts.drawerLoading() || userEdited || openedOnDraft) return + const cfg = opts.getCfg() + const deployed = opts.deployed() + if (cfg == null || deployed == null) return + // Snapshot before untracking: `getCfg` hands back the form's `$state` + // objects by reference, so only a deep read subscribes to the nested + // writes the form makes as it settles. + const settled = snapshotCfg(cfg) + untrack(() => { + if (cfgDiffers(settled, settledBaseline ?? deployed)) settledBaseline = settled + }) + }) + // Live "is there a local draft?" — the form diverges from the deployed // baseline. Gated on `!drawerLoading` (the baseline isn't settled yet // mid-load) and on a non-null baseline (a brand-new trigger has none, so // "unsaved changes" / discard-to-deployed is meaningless there). const hasDraft = $derived( - !opts.drawerLoading() && - opts.deployed() != null && - cfgDiffers(opts.getCfg() as Cfg, opts.deployed() as Cfg) + !opts.drawerLoading() && baseline != null && cfgDiffers(opts.getCfg() as Cfg, baseline as Cfg) ) // Reactive "banner is possible" — depends on `drawerLoading()` so it @@ -131,7 +187,7 @@ export function useTriggerDraftSync(opts: TriggerDraftSyncOptions): TriggerDraft if (opts.drawerLoading() || d == null) return untrack(() => { if (cfgDiffers(d, opts.getCfg() as Cfg)) { - void opts.applyCfg(d) + void opts.applyCfg(snapshotCfg(d)) } }) }) @@ -148,30 +204,32 @@ export function useTriggerDraftSync(opts: TriggerDraftSyncOptions): TriggerDraft discardTimer = undefined if (opts.drawerLoading()) return const cfg = opts.getCfg() - const deployed = opts.deployed() const h = handle if (!h || cfg == null) return - if (!cfgDiffers(cfg, deployed) && cfgDiffers(h.draft, deployed)) { - discard(opts.path(), deployed, true) + if (!cfgDiffers(cfg, baseline) && cfgDiffers(h.draft, baseline)) { + discard(opts.path(), baseline, true) } }, 600) } // persist-effect: form edits → handle; drop the draft when back at the - // deployed baseline. + // baseline. $effect(() => { if (opts.drawerLoading() || !opts.path()) return + // Nothing persists before the user's first input — the form's own + // settling is not an edit, and gating here rather than relying on the + // absorb-effect having run first keeps the two effects order-independent. + if (!userEdited && !openedOnDraft) return const cfg = opts.getCfg() if (cfg == null) return untrack(() => { const h = handle if (!h) return - const deployed = opts.deployed() - if (cfgDiffers(cfg, deployed)) { + if (cfgDiffers(cfg, baseline)) { if (cfgDiffers(cfg, h.draft)) h.draft = cfg - } else if (cfgDiffers(h.draft, deployed)) { - // Only when a draft actually exists to drop: `h.draft` equals - // `deployed` right after a discard or the post-load seed. + } else if (cfgDiffers(h.draft, baseline)) { + // Only when a draft actually exists to drop: `h.draft` equals the + // baseline right after a discard or the post-load seed. scheduleAutoDiscard() } }) @@ -202,7 +260,7 @@ export function useTriggerDraftSync(opts: TriggerDraftSyncOptions): TriggerDraft return hasBaseline }, get deployed() { - return opts.deployed() + return baseline }, get current() { return opts.getCfg() @@ -211,8 +269,14 @@ export function useTriggerDraftSync(opts: TriggerDraftSyncOptions): TriggerDraft const d = handle?.draft if (cfgDiffers(d, opts.getCfg() as Cfg)) { // Overlay the local autosave on the just-loaded backend config. - await opts.applyCfg(d) + await opts.applyCfg(snapshotCfg(d)) } + // The form is not rendered while the drawer loads, so anything that + // diverges from the deployed config right now is a draft restored onto + // it — by the overlay above, or by the editor from the backend before + // calling this — never the form settling. Absorbing that into the + // baseline would hide the user's own work behind a clean drawer. + openedOnDraft = cfgDiffers(opts.getCfg() as Cfg, opts.deployed()) // Adopt the post-load form state as the cell's baseline without // POSTing, consuming the entry's one-shot first-write seed guard. // Trigger drawers never write the cell programmatically on open, so @@ -221,14 +285,20 @@ export function useTriggerDraftSync(opts: TriggerDraftSyncOptions): TriggerDraft const p = opts.path() const cfg = opts.getCfg() if (ws && p && cfg != null) { - UserDraft.seed(opts.itemKind, p, structuredClone($state.snapshot(cfg)) as Cfg, { + UserDraft.seed(opts.itemKind, p, snapshotCfg(cfg) as Cfg, { workspace: ws }) } }, async resetToDeployed(path: string) { - const deployedCfg = structuredClone($state.snapshot(opts.deployed())) as Cfg + const deployedCfg = snapshotCfg(opts.deployed()) as Cfg discard(path, deployedCfg) + // Nothing of the user's is left in the form, so the gate closes again + // — otherwise the form settles on the schema's values a second time + // and the discarded draft comes straight back. + settledBaseline = undefined + userEdited = false + openedOnDraft = false await opts.applyCfg(deployedCfg) }, discard diff --git a/frontend/src/lib/userDraft.svelte.ts b/frontend/src/lib/userDraft.svelte.ts index 7942694f31..57e911dd3c 100644 --- a/frontend/src/lib/userDraft.svelte.ts +++ b/frontend/src/lib/userDraft.svelte.ts @@ -219,7 +219,10 @@ const DRAFT_COMPARE_IGNORED_FIELDS = [ 'version_id', 'parent_version', 'is_draft', - 'assets' + 'assets', + // Fixed at creation and absent from the resource editor's draft shape, so + // it only ever shows up on the deployed side of a comparison. + 'resource_type' ] as const /** diff --git a/frontend/src/lib/userDraftEditGate.ts b/frontend/src/lib/userDraftEditGate.ts new file mode 100644 index 0000000000..e3057a73ee --- /dev/null +++ b/frontend/src/lib/userDraftEditGate.ts @@ -0,0 +1,66 @@ +import { onDestroy } from 'svelte' + +/** + * What kind of event opened the gate. + * + * - `value`: the user changed something — the event *is* the edit. + * - `precursor`: a gesture that usually precedes an edit. Needed because a + * custom component (a picker, a toggle built out of divs) writes its value + * through Svelte state and fires no native value event at all, so waiting for + * one would drop those edits. The cost is that a bare click counts too. + */ +export type UserInputKind = 'value' | 'precursor' + +/** Events that ARE an edit. `drop` and `paste` matter on their own: text + * dragged in from another application produces no pointer or key event in this + * document at all. */ +const VALUE_EVENTS = ['input', 'change', 'drop', 'paste'] as const +/** `click` is here for the controls that mutate state from a click handler and + * fire no native value event — ArgInput's "Add item", say. A mouse always sends + * `pointerdown` first, but an assistive technology can activate one with a + * trusted `click` alone, and that is a real edit with nothing else to catch it. */ +const PRECURSOR_EVENTS = ['pointerdown', 'keydown', 'click'] as const + +/** + * A draft is supposed to record what the USER changed, but an editor built + * from a schema writes into the value on its own: the form materializes a + * property the stored item never carried (an empty string, `false`, the first + * option of a required enum, a schema `default`) and deletes one a `showExpr` + * hides. So merely opening an item whose schema has moved on makes it diverge + * from the deployed value with nobody having touched it — a draft nobody asked + * for, cluttering the workspace. + * + * An editor guards against that by gating its draft on this: nothing the form + * settles on counts until the user has actually put something in. Callers + * decide what a gate covers (the resource editor keys it by workspace, since + * switching workspaces re-renders the form against a fresh value) and what + * gating means for them — suspending the autosave, absorbing the settled value + * into the deployed baseline, or both. + * + * Capture phase puts this ahead of the handler that writes the value, so a gate + * opened here is already open by the time the edit lands. Listening on the + * document rather than the editor's own subtree is deliberate: pickers and + * modals render in portals outside it, and missing a real edit would silently + * drop the user's work, while opening the gate too eagerly only costs the + * phantom draft that existed before. + * + * Registers for the lifetime of the calling component — call it during init. + */ +export function onUserInput(handle: (kind: UserInputKind) => void): void { + if (typeof document === 'undefined') return + const listeners: Array<[string, (e: Event) => void]> = [] + const register = (type: string, kind: UserInputKind) => { + const onEvent = (e: Event) => { + // A programmatic `dispatchEvent` is untrusted, which is what keeps the + // form's own settling from opening the gate it is gated by. + if (e.isTrusted) handle(kind) + } + document.addEventListener(type, onEvent, true) + listeners.push([type, onEvent]) + } + for (const type of VALUE_EVENTS) register(type, 'value') + for (const type of PRECURSOR_EVENTS) register(type, 'precursor') + onDestroy(() => { + for (const [type, onEvent] of listeners) document.removeEventListener(type, onEvent, true) + }) +} diff --git a/frontend/src/lib/userDraftPrune.test.ts b/frontend/src/lib/userDraftPrune.test.ts new file mode 100644 index 0000000000..0276598d45 --- /dev/null +++ b/frontend/src/lib/userDraftPrune.test.ts @@ -0,0 +1,243 @@ +import { describe, it, expect, beforeEach, vi } from 'vitest' + +// The sweep DELETES drafts, so the guard that matters is which rows it picks. +// Stub the two collaborators it decides from — the draft listing and the +// per-kind diff — and assert on what it discards. +const listDrafts = vi.fn() +const getDraftDiffValues = vi.fn() +const updateDraft = vi.fn(async () => ({ status: 'saved', current_timestamp: 'x' })) + +vi.mock('./gen', () => ({ + DraftService: { + listDrafts: (...a: unknown[]) => listDrafts(...(a as [])), + updateDraft: (...a: unknown[]) => updateDraft(...(a as [])) + } +})) +// Only `getDraftDiffValues` is stubbed; `canDiffDraftKind` is the real one, so +// the kind filter is pinned against the actual overlay table. +vi.mock('./utils_draft_deploy', async (orig) => ({ + ...(await orig>()), + getDraftDiffValues: (...a: unknown[]) => getDraftDiffValues(...(a as [])) +})) +vi.mock('./localDraftHints.svelte', () => ({ setLocalDraftHint: vi.fn() })) +vi.mock('./workspaceDrafts.svelte', () => ({ invalidateWorkspaceDrafts: vi.fn() })) +const sendUserToast = vi.fn() +vi.mock('./toast', () => ({ sendUserToast: (...a: unknown[]) => sendUserToast(...(a as [])) })) + +// The sweep reads exactly one thing from the syncer — whether this tab is +// mid-write on the key — and writes nothing back to it. +let syncState = 'none' +vi.mock('./userDraftDbSyncer.svelte', () => ({ + UserDraftDbSyncer: { + getState: () => ({ + get state() { + return syncState + } + }) + } +})) + +let liveDraft = false +vi.mock('./userDraft.svelte', async (orig) => ({ + ...(await orig>()), + UserDraft: { has: () => liveDraft } +})) + +import { pruneMeaninglessDrafts } from './userDraftPrune' + +const row = (over: Record = {}) => ({ + kind: 'resource', + path: 'u/me/r', + draft_only: false, + legacy_draft: false, + mine: true, + can_write: true, + created_at: '2026-01-01T00:00:00Z', + ...over +}) +const diff = (over: Record = {}) => ({ + deployed: { value: { host: 'h' } }, + draft: { value: { host: 'h' } }, + hasDraft: true, + noDeployed: false, + ...over +}) +const discardedPaths = () => updateDraft.mock.calls.map((c: any[]) => c[0].path as string) + +beforeEach(() => { + localStorage.clear() + vi.clearAllMocks() + updateDraft.mockResolvedValue({ status: 'saved', current_timestamp: 'x' }) + syncState = 'none' + liveDraft = false +}) + +describe('pruneMeaninglessDrafts', () => { + it('discards a draft whose diff against the deployed value is empty', async () => { + listDrafts.mockResolvedValue([row()]) + getDraftDiffValues.mockResolvedValue(diff()) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(discardedPaths()).toEqual(['u/me/r']) + }) + + it('keeps a draft that carries a real change', async () => { + listDrafts.mockResolvedValue([row()]) + getDraftDiffValues.mockResolvedValue(diff({ draft: { value: { host: 'other' } } })) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(updateDraft).not.toHaveBeenCalled() + }) + + it('never touches a draft-only item — the draft is the whole item', async () => { + listDrafts.mockResolvedValue([row({ draft_only: true })]) + getDraftDiffValues.mockResolvedValue(diff()) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(updateDraft).not.toHaveBeenCalled() + }) + + it('never touches another user’s row, or one it cannot write', async () => { + listDrafts.mockResolvedValue([row({ mine: false }), row({ path: 'u/me/b', can_write: false })]) + getDraftDiffValues.mockResolvedValue(diff()) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(updateDraft).not.toHaveBeenCalled() + }) + + it('leaves a draft alone when its diff cannot be fetched, and retries it later', async () => { + listDrafts.mockResolvedValue([row()]) + getDraftDiffValues.mockRejectedValue(new Error('boom')) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(updateDraft).not.toHaveBeenCalled() + // A row that could not be judged is not a row that carries changes, so + // the pass must stay open rather than strand it. + getDraftDiffValues.mockResolvedValue(diff()) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(discardedPaths()).toEqual(['u/me/r']) + }) + + it('conditions the delete on the timestamp it judged, so a row that moved is spared', async () => { + listDrafts.mockResolvedValue([row()]) + getDraftDiffValues.mockResolvedValue(diff()) + updateDraft.mockResolvedValue({ status: 'conflict', current_timestamp: 'newer' }) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(updateDraft).toHaveBeenCalledWith({ + workspace: 'main', + kind: 'resource', + path: 'u/me/r', + requestBody: { value: null, last_sync: '2026-01-01T00:00:00Z', force: false } + }) + // Refused, so nothing is reported as cleared — and the sweep leaves no + // state behind for the editor's own autosave to trip over. + expect(sendUserToast).not.toHaveBeenCalled() + }) + + it('does not keep retrying a row the server will never judge', async () => { + listDrafts.mockResolvedValue([row()]) + getDraftDiffValues.mockRejectedValue({ status: 404 }) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(updateDraft).not.toHaveBeenCalled() + // Sealed: a 4xx is final, unlike the transient case above. + listDrafts.mockResolvedValue([row()]) + getDraftDiffValues.mockResolvedValue(diff()) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(updateDraft).not.toHaveBeenCalled() + }) + + it('gives up after a bounded number of unresolved passes', async () => { + listDrafts.mockResolvedValue([row()]) + getDraftDiffValues.mockRejectedValue(new Error('network')) + for (let i = 0; i < 3; i++) await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(listDrafts).toHaveBeenCalledTimes(3) + // Sealed on the third: an unresolvable row cannot re-list forever. + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(listDrafts).toHaveBeenCalledTimes(3) + }) + + it('skips a kind no diff can be computed for, and still seals', async () => { + listDrafts.mockResolvedValue([row({ kind: 'trigger_webhook', path: 'u/me/hook' })]) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(getDraftDiffValues).not.toHaveBeenCalled() + // Unjudgeable is permanent, not transient: leaving the pass open would + // re-run the sweep on every page load forever. + listDrafts.mockResolvedValue([row()]) + getDraftDiffValues.mockResolvedValue(diff()) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(updateDraft).not.toHaveBeenCalled() + }) + + it('leaves alone a draft this tab is editing', async () => { + liveDraft = true + listDrafts.mockResolvedValue([row()]) + getDraftDiffValues.mockResolvedValue(diff()) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(updateDraft).not.toHaveBeenCalled() + }) + + it('leaves alone a draft with a write queued or in flight', async () => { + syncState = 'pending' + listDrafts.mockResolvedValue([row()]) + getDraftDiffValues.mockResolvedValue(diff()) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(updateDraft).not.toHaveBeenCalled() + }) + + it('does not count, or seal the pass on, a delete that failed to send', async () => { + listDrafts.mockResolvedValue([row()]) + getDraftDiffValues.mockResolvedValue(diff()) + updateDraft.mockRejectedValueOnce(new Error('network')) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(sendUserToast).not.toHaveBeenCalled() + // The pass stayed open, so the draft left behind is retried. + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(discardedPaths()).toEqual(['u/me/r', 'u/me/r']) + }) + + it('never touches a legacy workspace-level row', async () => { + listDrafts.mockResolvedValue([row({ legacy_draft: true })]) + getDraftDiffValues.mockResolvedValue(diff()) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(updateDraft).not.toHaveBeenCalled() + }) + + it('only sweeps the kinds whose editors are gated', async () => { + listDrafts.mockResolvedValue([ + row({ kind: 'script', path: 'u/me/s' }), + row({ kind: 'flow', path: 'u/me/f' }), + row({ kind: 'app', path: 'u/me/a' }), + row({ kind: 'trigger_schedule', path: 'u/me/sched' }), + row() + ]) + getDraftDiffValues.mockResolvedValue(diff()) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(discardedPaths().sort()).toEqual(['u/me/r', 'u/me/sched']) + // The expensive payload fetches are never made for the ungated kinds. + expect(getDraftDiffValues).toHaveBeenCalledTimes(2) + }) + + it('leaves the pass open when a row was skipped as busy', async () => { + liveDraft = true + listDrafts.mockResolvedValue([row()]) + getDraftDiffValues.mockResolvedValue(diff()) + await pruneMeaninglessDrafts('main', 'me@x.dev') + liveDraft = false + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(discardedPaths()).toEqual(['u/me/r']) + }) + + it('runs once per workspace and user', async () => { + listDrafts.mockResolvedValue([row()]) + getDraftDiffValues.mockResolvedValue(diff()) + await pruneMeaninglessDrafts('main', 'me@x.dev') + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(updateDraft).toHaveBeenCalledTimes(1) + await pruneMeaninglessDrafts('other', 'me@x.dev') + expect(updateDraft).toHaveBeenCalledTimes(2) + }) + + it('retries next mount when the listing failed', async () => { + listDrafts.mockRejectedValueOnce(new Error('offline')) + await pruneMeaninglessDrafts('main', 'me@x.dev') + listDrafts.mockResolvedValue([row()]) + getDraftDiffValues.mockResolvedValue(diff()) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(discardedPaths()).toEqual(['u/me/r']) + }) +}) diff --git a/frontend/src/lib/userDraftPrune.ts b/frontend/src/lib/userDraftPrune.ts new file mode 100644 index 0000000000..465048d09d --- /dev/null +++ b/frontend/src/lib/userDraftPrune.ts @@ -0,0 +1,230 @@ +/** + * One-off sweep that drops drafts carrying no changes. + * + * `onUserInput` stops new ones from being written; the ones already stored need + * this pass to clear. Runs once per (workspace, user) per browser, after the + * localStorage→DB migration so anything it just uploaded is swept too. + * + * Scoped to the kinds whose editors that gate covers. A script, flow or app + * draft only ever came from an explicit edit, so there is no phantom to clear + * there — and `getDraftDiffValues` would fetch each one's full deployed payload + * at login to prove it. + * + * A draft is dropped only when the diff the user would be shown is empty: both + * sides come from `getDraftDiffValues`, the same canonicalization the diff + * drawer renders, compared with the same `draftValuesEqual` the editors use. + * Anything that can't be established is left alone — a `draft_only` item (no + * deployed counterpart, so discarding would destroy the item itself), a kind + * with no diff support, a failed fetch, and the legacy workspace-level rows, + * which belong to nobody and are admin-gated to migrate. + * + * Deleting is the dangerous half, and the equality behind it is always stale: + * it was read one round trip ago, and every candidate is read before any is + * deleted. So the delete is a compare-and-delete — `last_sync` is the + * timestamp the row was judged on, and the backend drops it only if nothing has + * written it since, whoever wrote it. + * + * It is sent straight to `DraftService`, NOT through `UserDraftDbSyncer`. That + * syncer exists to autosave an editor's own live value, and everything it does + * for that — parking the payload for the `pagehide` flush, debouncing, holding + * a per-tab `last_sync` baseline and conflict state — is a way for a one-shot + * delete to reach back into whatever the user is doing in the same tab. This + * sweep wants exactly one conditional request and no state afterwards. + */ + +import { DraftService } from './gen' +import type { UserDraftItemKind } from './gen' +import { sendUserToast } from './toast' +import { setLocalDraftHint } from './localDraftHints.svelte' +import { UserDraft, draftValuesEqual } from './userDraft.svelte' +import { UserDraftDbSyncer } from './userDraftDbSyncer.svelte' +import { canDiffDraftKind, getDraftDiffValues } from './utils_draft_deploy' +import { invalidateWorkspaceDrafts } from './workspaceDrafts.svelte' + +const SENTINEL_PREFIX = 'userdraft/pruned/v1/' + +/** A pass that leaves anything unresolved runs again next mount, which costs a + * listing plus an overlay GET per row. Bounded so no permanently-unresolvable + * row can make that repeat forever — whatever the reason it can't be judged. */ +const MAX_PASSES = 3 + +/** The editors whose forms are built from a schema, and so the only kinds that + * could have banked a draft nobody wrote — minus the ones no diff can be + * computed for, which would throw and keep the pass unsealed forever. */ +function isSweepableKind(kind: UserDraftItemKind): boolean { + return (kind === 'resource' || kind.startsWith('trigger_')) && canDiffDraftKind(kind) +} + +/** Overlay GETs are one round trip each and a cluttered workspace has dozens; + * a small window keeps the sweep off the critical path of a fresh login. */ +const CONCURRENCY = 4 + +/** Guards against the layout effect firing again before the sentinel lands. */ +const inFlight = new Set() + +type Candidate = { + kind: UserDraftItemKind + path: string + /** The row's `created_at` as listed — the baseline the delete is conditioned on. */ + createdAt: string +} + +const attemptsKey = (sentinel: string) => `${sentinel}:attempts` + +function readAttempts(sentinel: string): number { + try { + const n = Number(localStorage.getItem(attemptsKey(sentinel))) + return Number.isFinite(n) && n > 0 ? n : 0 + } catch { + return 0 + } +} + +/** Is this tab holding or writing this draft right now? */ +function busyLocally(workspace: string, kind: UserDraftItemKind, path: string): boolean { + if (UserDraft.has(kind, path, { workspace })) return true + return UserDraftDbSyncer.getState({ workspace, itemKind: kind, path }).state !== 'none' +} + +/** A 4xx is the server's final answer for this row — the item is gone, or the + * kind's overlay endpoint isn't served by this build (a feature-gated trigger + * on CE). Retrying it on every page load would never succeed. Anything else + * (network, 5xx) is worth another pass. 429 asks for exactly that. */ +function isPermanentlyUnjudgeable(e: unknown): boolean { + const status = (e as { status?: unknown })?.status + return typeof status === 'number' && status >= 400 && status < 500 && status !== 429 +} + +/** `undefined` when the diff could not be fetched and might be next time — + * distinct from `false`, so the caller can leave the pass open rather than + * strand a row it never judged. */ +async function carriesNoChanges( + workspace: string, + { kind, path }: Candidate +): Promise { + try { + const { deployed, draft, hasDraft, noDeployed } = await getDraftDiffValues( + kind, + path, + workspace + ) + // `hasDraft` false means the overlay had no draft row and the item's own + // value stood in for the draft side — there is nothing to discard, and the + // two sides would compare equal by construction. + if (!hasDraft || noDeployed) return false + return draftValuesEqual(draft, deployed) + } catch (e) { + return isPermanentlyUnjudgeable(e) ? false : undefined + } +} + +async function mapWithLimit( + items: T[], + limit: number, + fn: (item: T) => Promise +): Promise { + const out = new Array(items.length) + let next = 0 + await Promise.all( + Array.from({ length: Math.min(limit, items.length) }, async () => { + while (next < items.length) { + const i = next++ + out[i] = await fn(items[i]) + } + }) + ) + return out +} + +export async function pruneMeaninglessDrafts(workspace: string, userKey: string): Promise { + if (typeof localStorage === 'undefined') return + const sentinel = `${SENTINEL_PREFIX}${workspace}/${userKey}` + if (inFlight.has(sentinel)) return + try { + if (localStorage.getItem(sentinel)) return + } catch { + // Storage unavailable (private mode): the sweep can't record that it ran, + // and re-running it on every mount would cost an overlay GET per draft. + return + } + inFlight.add(sentinel) + try { + const rows = await DraftService.listDrafts({ workspace }) + // Anything left unresolved keeps the pass open: a row skipped as busy was + // never judged, and one whose delete failed is still there. Sealing on + // either would strand it. + let unresolved = 0 + const candidates: Candidate[] = rows + // `draft_only` rows ARE the item; `mine` / `can_write` are the same + // gate the discard endpoint enforces, so anything else would 403. + .filter((r) => !r.draft_only && r.mine && r.can_write) + .filter((r) => isSweepableKind(r.kind) && !r.legacy_draft) + .filter((r) => { + if (!busyLocally(workspace, r.kind, r.path)) return true + unresolved++ + return false + }) + .map((r) => ({ + kind: r.kind, + path: r.path, + createdAt: r.created_at + })) + + const empty: Candidate[] = [] + await mapWithLimit(candidates, CONCURRENCY, async (c) => { + const verdict = await carriesNoChanges(workspace, c) + if (verdict === undefined) unresolved++ + else if (verdict) empty.push(c) + }) + + let discarded = 0 + for (const c of empty) { + // Re-check: the reads above took a while, and the user may have opened + // this item in the meantime. + if (busyLocally(workspace, c.kind, c.path)) { + unresolved++ + continue + } + try { + const resp = await DraftService.updateDraft({ + workspace, + kind: c.kind, + path: c.path, + requestBody: { value: null, last_sync: c.createdAt, force: false } + }) + // `conflict` means the row moved past the timestamp we judged it on, + // so it is no longer the empty draft we decided to drop. + if (resp.status === 'saved') { + setLocalDraftHint(workspace, c.kind, c.path, false) + discarded++ + } + } catch { + unresolved++ + } + } + if (discarded > 0) { + invalidateWorkspaceDrafts(workspace) + sendUserToast(`Cleared ${discarded} draft${discarded > 1 ? 's' : ''} that carried no changes`) + } + // Seal once nothing is left hanging, or once we have tried enough times + // that whatever is hanging is not going to resolve. + const attempts = readAttempts(sentinel) + 1 + if (unresolved === 0 || attempts >= MAX_PASSES) { + try { + localStorage.setItem(sentinel, new Date().toISOString()) + localStorage.removeItem(attemptsKey(sentinel)) + } catch { + // Nothing to do — the pass is idempotent, it just runs again. + } + } else { + try { + localStorage.setItem(attemptsKey(sentinel), String(attempts)) + } catch {} + } + } catch { + // Fire-and-forget from the layout: a workspace whose draft list can't be + // read is left exactly as it was. + } finally { + inFlight.delete(sentinel) + } +} diff --git a/frontend/src/lib/utils_draft_deploy.ts b/frontend/src/lib/utils_draft_deploy.ts index f322bb27b9..0d121d0627 100644 --- a/frontend/src/lib/utils_draft_deploy.ts +++ b/frontend/src/lib/utils_draft_deploy.ts @@ -95,6 +95,19 @@ const OVERLAY_GETTERS: Partial< EmailTriggerService.getEmailTrigger({ workspace, path, getDraft: true }) } +/** Whether `getDraftDiffValues` can produce a diff for this kind at all. The + * script/flow/app family is handled inline; every other kind needs an overlay + * getter and throws without one — several trigger kinds have none. */ +export function canDiffDraftKind(kind: DraftKind): boolean { + return ( + kind === 'script' || + kind === 'flow' || + kind === 'app' || + kind === 'raw_app' || + OVERLAY_GETTERS[kind] !== undefined + ) +} + /** Strip the per-user draft-overlay metadata, returning `{deployed, draft}`. */ function splitOverlay(r: any): { deployed: any diff --git a/frontend/src/routes/(root)/(logged)/+layout.svelte b/frontend/src/routes/(root)/(logged)/+layout.svelte index a20c61f2ff..e82d328c76 100644 --- a/frontend/src/routes/(root)/(logged)/+layout.svelte +++ b/frontend/src/routes/(root)/(logged)/+layout.svelte @@ -74,6 +74,7 @@ import { createUsageResources, registerUsageResources } from '$lib/usage.svelte' import { purgeLegacyUserDrafts } from '$lib/userDraftLegacyMigration' import { migrateUserDraftsToDb } from '$lib/userDraftDbMigration' + import { pruneMeaninglessDrafts } from '$lib/userDraftPrune' import DraftMigrationErrorModal from '$lib/components/DraftMigrationErrorModal.svelte' import { onDestroy, setContext, untrack } from 'svelte' import { base } from '$app/paths' @@ -792,12 +793,16 @@ // drafts). `migrateUserDraftsToDb` then pushes the workspace-scoped // `userdraft/w/{ws}/{kind}/{path}` keys — written by the editor with the // correct workspace — onto the server-side draft table, clearing LS on - // success. + // success. `pruneMeaninglessDrafts` then clears the drafts an older, stricter + // comparison saved for changes nobody made; it runs after the upload so the + // entries that just landed are swept in the same pass. $effect(() => { - if ($workspaceStore && $userStore) { + const ws = $workspaceStore + const email = $userStore?.email + if (ws && email) { untrack(() => { purgeLegacyUserDrafts() - void migrateUserDraftsToDb() + void migrateUserDraftsToDb().then(() => pruneMeaninglessDrafts(ws, email)) }) } }) From a9d42b489f06261919ba01e8221272e3ffcf6b22 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 7 Sep 2026 18:20:28 +0000 Subject: [PATCH 015/144] chore(main): release 1.805.0 (#10995) * chore(main): release 1.805.0 * Apply automatic changes --------- Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com> --- .release-please-manifest.json | 2 +- CHANGELOG.md | 25 +++ backend/Cargo.lock | 202 +++++++++--------- backend/Cargo.toml | 4 +- .../parsers/windmill-parser-wasm/Cargo.lock | 48 ++--- .../parsers/windmill-parser-wasm/Cargo.toml | 2 +- backend/windmill-api/openapi.yaml | 2 +- benchmarks/lib.ts | 2 +- cli/src/core/constants.ts | 2 +- frontend/package-lock.json | 4 +- frontend/package.json | 2 +- lsp/Pipfile | 2 +- openflow.openapi.yaml | 2 +- .../WindmillClient/WindmillClient.psd1 | 2 +- python-client/wmill/pyproject.toml | 2 +- typescript-client/jsr.json | 2 +- typescript-client/package.json | 2 +- version.txt | 2 +- windmill-yaml-validator/package-lock.json | 4 +- windmill-yaml-validator/package.json | 2 +- 20 files changed, 170 insertions(+), 145 deletions(-) diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 5dc2004c6f..f4fac9d2e0 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "1.804.0" + ".": "1.805.0" } diff --git a/CHANGELOG.md b/CHANGELOG.md index 4acfb0edad..f431b9fd96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,30 @@ # Changelog +## [1.805.0](https://github.com/windmill-labs/windmill/compare/v1.804.0...v1.805.0) (2026-09-07) + + +### Features + +* **git-sync:** sync extra_perms for variables ([#11004](https://github.com/windmill-labs/windmill/issues/11004)) ([ee9e550](https://github.com/windmill-labs/windmill/commit/ee9e550a484fda286eeab43b7db5f314b8b2d0d9)) +* go to referenced row from foreign-keyed cells in the database manager ([#10998](https://github.com/windmill-labs/windmill/issues/10998)) ([e2b63d1](https://github.com/windmill-labs/windmill/commit/e2b63d177ae4e5c980cb5da34154540c90771b63)) +* let `// materialize` declare a `dbt://` warehouse-relation write ([#10978](https://github.com/windmill-labs/windmill/issues/10978)) ([c6e0302](https://github.com/windmill-labs/windmill/commit/c6e0302d7c1c60147f19d55a3923be8b1aa99c9c)) +* report resource type picks to the hub and rank pickers by popularity ([#10982](https://github.com/windmill-labs/windmill/issues/10982)) ([48a5615](https://github.com/windmill-labs/windmill/commit/48a56158c135c3b13a02f73b7b8438bc691f85b4)) +* run a linked AI agent's draft when testing a flow, and offer to deploy it ([#10993](https://github.com/windmill-labs/windmill/issues/10993)) ([7feaf61](https://github.com/windmill-labs/windmill/commit/7feaf619cf0ec021d66be14ef535cf2149bec58a)) +* show the new-tab icon on a chat path pill while the modifier is held ([#10976](https://github.com/windmill-labs/windmill/issues/10976)) ([5da4ea4](https://github.com/windmill-labs/windmill/commit/5da4ea43fbd01e43aa14e75dc597d7ce5d8797ab)) + + +### Bug Fixes + +* **cli:** keep permissioned_as on single-item push, as sync push does ([#11000](https://github.com/windmill-labs/windmill/issues/11000)) ([5f3f99b](https://github.com/windmill-labs/windmill/commit/5f3f99ba6915b7c5df663a30b35f4cd02050e728)) +* **cli:** say which workspace id is targeted, and when wmill.yaml is bypassed ([#11006](https://github.com/windmill-labs/windmill/issues/11006)) ([7643e9b](https://github.com/windmill-labs/windmill/commit/7643e9bd77c56f72596b8dca50801baf58984198)) +* **frontend:** no phantom draft when opening a CLI-pushed script ([#10997](https://github.com/windmill-labs/windmill/issues/10997)) ([1be390a](https://github.com/windmill-labs/windmill/commit/1be390aa878e15a58f530f3a878e8f9caeb89c43)) +* **frontend:** stop hover flicker on asset nodes shared with an overflow popover ([#10996](https://github.com/windmill-labs/windmill/issues/10996)) ([519a5c8](https://github.com/windmill-labs/windmill/commit/519a5c8bc70b44a7417e83c26c7b9c58b2c4fb9c)) +* let a draft-only schedule, trigger or resource be deleted ([#11010](https://github.com/windmill-labs/windmill/issues/11010)) ([8d0f475](https://github.com/windmill-labs/windmill/commit/8d0f4754e4e0c78696ee0c97ff2de3016ece3bac)) +* point the app viewer's edit button at the editor for the app's kind ([#11009](https://github.com/windmill-labs/windmill/issues/11009)) ([8f553ea](https://github.com/windmill-labs/windmill/commit/8f553eab353103fd8a28a00532e1766f133590de)) +* seed runs page filter defaults through the url so they survive sync ([#11005](https://github.com/windmill-labs/windmill/issues/11005)) ([f381acd](https://github.com/windmill-labs/windmill/commit/f381acdb37f66f5e272bc37938e69f734987d53f)) +* stop an untouched item's form from saving a draft nobody wrote ([#10964](https://github.com/windmill-labs/windmill/issues/10964)) ([c3f7f8a](https://github.com/windmill-labs/windmill/commit/c3f7f8a45830fb548aa628ebf6e2b6c95c6de67f)) +* write and read python job files as utf-8, not the platform locale ([#10994](https://github.com/windmill-labs/windmill/issues/10994)) ([670404f](https://github.com/windmill-labs/windmill/commit/670404ffe27fedc3858b46b0c6b3312fbe175e13)) + ## [1.804.0](https://github.com/windmill-labs/windmill/compare/v1.803.0...v1.804.0) (2026-09-05) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index e4dc875e49..9a458a8713 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -2763,18 +2763,18 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.16" +version = "0.5.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e" +checksum = "98b0cc327b5bc766e7fda9c9260cc0fa81b43a8e240440422dff70788e3f9ef1" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-deque" -version = "0.8.7" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a" dependencies = [ "crossbeam-epoch", "crossbeam-utils", @@ -2782,27 +2782,27 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.20" +version = "0.9.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-queue" -version = "0.3.13" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "803d13fb3b09d88be9f4dbc29062c66b19bf7170867ceb746d2a8689bf6c7a26" +checksum = "03e8bd762f7479489c70ed6c768ddca99d7296857de437a68dcb2a94365b3fae" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.22" +version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" [[package]] name = "crossterm_winapi" @@ -6640,9 +6640,9 @@ dependencies = [ [[package]] name = "io-uring" -version = "0.7.14" +version = "0.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d64d8ca234d152948ceaede1f419b6a83983a5ecccaac05fb337a809c96d3aa6" +checksum = "ed3bd0ecfbb87805f538bb7b32e5239ca0763890c623e349860ecba69469f2bb" dependencies = [ "bitflags 2.13.1", "cfg-if", @@ -6664,9 +6664,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.12.1" +version = "2.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78" +checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" [[package]] name = "ipnetwork" @@ -9077,9 +9077,9 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pest" -version = "2.9.0" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a07a60cc7a4d00c91f95c685609d1d2f79050e6804b70ebedd7650f0b839bcf" +checksum = "6d45aeb61b4bf818e12d4205f2466f8c4748f85f4fce0146d1c03d69d753f0ad" dependencies = [ "memchr", "ucd-trie", @@ -9087,9 +9087,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.9.0" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3a83744a5c8455b8b3e0dc5031362780a347c878bdd11584d1a8984228cc88d" +checksum = "89cc5a242e25ed4e7704d0be240f2cfbe20a8c27e7e252d94835be93d92dc39f" dependencies = [ "pest", "pest_generator", @@ -9097,9 +9097,9 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.9.0" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0cd3451aa3de60d4b9a1e736885e4dea6b31617598026f12256ad566d63304a" +checksum = "7abf21475cc3820fe4b2ca2dc2142902f67a02189f3b5b3a229f4febc01a43e5" dependencies = [ "pest", "pest_meta", @@ -9110,9 +9110,9 @@ dependencies = [ [[package]] name = "pest_meta" -version = "2.9.0" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e04d3a0849e241d7dfce834c83b1c5edc8622009e8dd51a12ba1927c32f05496" +checksum = "adba4db388f687393c18c51348d44a41d870ca9df71a2c98172ea3035dc6936e" dependencies = [ "pest", ] @@ -14747,7 +14747,7 @@ dependencies = [ [[package]] name = "windmill" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-nats", @@ -14835,7 +14835,7 @@ dependencies = [ [[package]] name = "windmill-ai" -version = "1.804.0" +version = "1.805.0" dependencies = [ "async-stream", "async-trait", @@ -14868,7 +14868,7 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.804.0" +version = "1.805.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14881,7 +14881,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "argon2", @@ -15021,7 +15021,7 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.804.0" +version = "1.805.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15044,7 +15044,7 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.804.0" +version = "1.805.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15061,7 +15061,7 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15087,7 +15087,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.804.0" +version = "1.805.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -15097,7 +15097,7 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.804.0" +version = "1.805.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15114,7 +15114,7 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.804.0" +version = "1.805.0" dependencies = [ "axum 0.8.9", "base64 0.22.1", @@ -15136,7 +15136,7 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15159,7 +15159,7 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.804.0" +version = "1.805.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15175,7 +15175,7 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.804.0" +version = "1.805.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15197,7 +15197,7 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.804.0" +version = "1.805.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15218,7 +15218,7 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.804.0" +version = "1.805.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15232,7 +15232,7 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-nats", @@ -15267,7 +15267,7 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15292,7 +15292,7 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.804.0" +version = "1.805.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15320,7 +15320,7 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15342,7 +15342,7 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.804.0" +version = "1.805.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15362,7 +15362,7 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.804.0" +version = "1.805.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15400,7 +15400,7 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15428,7 +15428,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.804.0" +version = "1.805.0" dependencies = [ "lazy_static", "serde", @@ -15440,7 +15440,7 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.804.0" +version = "1.805.0" dependencies = [ "argon2", "axum 0.8.9", @@ -15464,7 +15464,7 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.804.0" +version = "1.805.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15478,7 +15478,7 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.804.0" +version = "1.805.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15513,7 +15513,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.804.0" +version = "1.805.0" dependencies = [ "chrono", "lazy_static", @@ -15527,7 +15527,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15546,7 +15546,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.804.0" +version = "1.805.0" dependencies = [ "aes-gcm", "aho-corasick", @@ -15652,7 +15652,7 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.804.0" +version = "1.805.0" dependencies = [ "chrono", "futures", @@ -15672,7 +15672,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.804.0" +version = "1.805.0" dependencies = [ "regex", "serde", @@ -15687,7 +15687,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "astral-tokio-tar", @@ -15714,7 +15714,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "futures", @@ -15731,7 +15731,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.804.0" +version = "1.805.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -15747,7 +15747,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-trait", @@ -15768,7 +15768,7 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-trait", @@ -15799,7 +15799,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "arc-swap", @@ -15824,7 +15824,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-stream", @@ -15858,7 +15858,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "futures", @@ -15876,7 +15876,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.804.0" +version = "1.805.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -15885,7 +15885,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "lazy_static", @@ -15897,7 +15897,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "serde_json", @@ -15909,7 +15909,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "gosyn", @@ -15921,7 +15921,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "lazy_static", @@ -15933,7 +15933,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "serde_json", @@ -15945,7 +15945,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "nu-parser", @@ -15956,7 +15956,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15967,7 +15967,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15979,7 +15979,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "rustpython-ast", @@ -15990,7 +15990,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-recursion", @@ -16012,7 +16012,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "serde_json", @@ -16024,7 +16024,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "lazy_static", @@ -16038,7 +16038,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -16055,7 +16055,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "lazy_static", @@ -16068,7 +16068,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "serde", @@ -16080,7 +16080,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "lazy_static", @@ -16098,7 +16098,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -16114,7 +16114,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "rustpython-ast", @@ -16130,7 +16130,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "lazy_static", @@ -16144,7 +16144,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-recursion", @@ -16183,7 +16183,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "const_format", @@ -16223,7 +16223,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.804.0" +version = "1.805.0" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -16234,7 +16234,7 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-recursion", @@ -16269,7 +16269,7 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-trait", @@ -16293,7 +16293,7 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-trait", @@ -16326,7 +16326,7 @@ dependencies = [ [[package]] name = "windmill-trigger-amqp" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-trait", @@ -16353,7 +16353,7 @@ dependencies = [ [[package]] name = "windmill-trigger-azure" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-trait", @@ -16386,7 +16386,7 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-trait", @@ -16406,7 +16406,7 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-trait", @@ -16440,7 +16440,7 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-trait", @@ -16476,7 +16476,7 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-trait", @@ -16499,7 +16499,7 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-trait", @@ -16523,7 +16523,7 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-nats", @@ -16547,7 +16547,7 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-trait", @@ -16582,7 +16582,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-trait", @@ -16610,7 +16610,7 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-trait", @@ -16635,7 +16635,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "bitflags 2.13.1", @@ -16654,7 +16654,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-once-cell", @@ -16771,7 +16771,7 @@ dependencies = [ [[package]] name = "windmill-worker-volumes" -version = "1.804.0" +version = "1.805.0" dependencies = [ "bytes", "futures", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 7abb3b92ee..f0bccb6fa8 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.804.0" +version = "1.805.0" authors.workspace = true edition.workspace = true @@ -88,7 +88,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"] [workspace.package] -version = "1.804.0" +version = "1.805.0" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/parsers/windmill-parser-wasm/Cargo.lock b/backend/parsers/windmill-parser-wasm/Cargo.lock index 9ed900214c..13597b06c9 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.lock +++ b/backend/parsers/windmill-parser-wasm/Cargo.lock @@ -6191,7 +6191,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill-common" -version = "1.804.0" +version = "1.805.0" dependencies = [ "aho-corasick", "anyhow", @@ -6274,7 +6274,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.804.0" +version = "1.805.0" dependencies = [ "proc-macro2", "quote", @@ -6286,7 +6286,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.804.0" +version = "1.805.0" dependencies = [ "convert_case", "serde", @@ -6295,7 +6295,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "lazy_static", @@ -6307,7 +6307,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "serde_json", @@ -6319,7 +6319,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "gosyn", @@ -6331,7 +6331,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "lazy_static", @@ -6343,7 +6343,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "serde_json", @@ -6355,7 +6355,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "nu-parser", @@ -6366,7 +6366,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6377,7 +6377,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6389,7 +6389,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6400,7 +6400,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-recursion", @@ -6422,7 +6422,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "serde_json", @@ -6434,7 +6434,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "lazy_static", @@ -6448,7 +6448,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "convert_case", @@ -6465,7 +6465,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "lazy_static", @@ -6478,7 +6478,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "serde", @@ -6490,7 +6490,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "lazy_static", @@ -6508,7 +6508,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -6524,7 +6524,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6540,7 +6540,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "getrandom 0.2.17", @@ -6572,7 +6572,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "lazy_static", @@ -6586,7 +6586,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "bitflags", diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index cbb7102820..7de860c637 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -12,7 +12,7 @@ resolver = "2" members = ["."] [workspace.package] -version = "1.804.0" +version = "1.805.0" edition = "2021" authors = ["Ruben Fiszel "] diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index aeb7d27f36..5c0a6d57c6 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.804.0 + version: 1.805.0 title: Windmill API contact: diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 12930e8de4..2899ff694c 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.804.0"; +export const VERSION = "v1.805.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/src/core/constants.ts b/cli/src/core/constants.ts index 35bd6bc2e7..b5494b112c 100644 --- a/cli/src/core/constants.ts +++ b/cli/src/core/constants.ts @@ -10,4 +10,4 @@ export const WM_FORK_PREFIX = "wm-fork"; // (e.g. utils.ts) can read it without importing main.ts and creating a circular // dependency (main → workspace → utils → main) that triggers a TDZ. // Re-exported from main.ts for backwards compatibility. -export const VERSION = "1.804.0"; +export const VERSION = "1.805.0"; diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 7dde8c270e..a200c878e3 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "@windmill-labs/components", - "version": "1.804.0", + "version": "1.805.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@windmill-labs/components", - "version": "1.804.0", + "version": "1.805.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { diff --git a/frontend/package.json b/frontend/package.json index 593ad0e404..d96e512c00 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "@windmill-labs/components", - "version": "1.804.0", + "version": "1.805.0", "scripts": { "dev": "vite dev", "dev:ui-builder": "mv static/ui_builder static/ui_builder.dev-disabled 2>/dev/null || true ; trap 'mv static/ui_builder.dev-disabled static/ui_builder 2>/dev/null || true' EXIT ; vite dev", diff --git a/lsp/Pipfile b/lsp/Pipfile index 0015ac3510..e1f33d70f0 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,7 +4,7 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.804.0" +wmill = ">=1.805.0" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index 3e2a3950f4..0bbc5a7a37 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: '3.0.3' info: - version: 1.804.0 + version: 1.805.0 title: OpenFlow Spec contact: name: Ruben Fiszel diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index aa3bfc30a3..1068d7b56c 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. - ModuleVersion = '1.804.0' + ModuleVersion = '1.805.0' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index 39d53054cb..d00cd3223f 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.804.0" +version = "1.805.0" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index 7d5452bf42..abbf63201c 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.804.0", + "version": "1.805.0", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./sqlUtils.ts", "./client.ts", "./wacError.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index decf7d342f..1d3542703d 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.804.0", + "version": "1.805.0", "author": "Ruben Fiszel", "license": "Apache 2.0", "homepage": "https://github.com/windmill-labs/windmill/tree/main/typescript-client#readme", diff --git a/version.txt b/version.txt index 494531e9d9..c746f61dea 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.804.0 +1.805.0 diff --git a/windmill-yaml-validator/package-lock.json b/windmill-yaml-validator/package-lock.json index fd43bd29a1..cc63f3a090 100644 --- a/windmill-yaml-validator/package-lock.json +++ b/windmill-yaml-validator/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-yaml-validator", - "version": "1.804.0", + "version": "1.805.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-yaml-validator", - "version": "1.804.0", + "version": "1.805.0", "license": "Apache 2.0", "dependencies": { "@stoplight/yaml": "^4.3.0", diff --git a/windmill-yaml-validator/package.json b/windmill-yaml-validator/package.json index da7c582f9e..d88bcdcf0c 100644 --- a/windmill-yaml-validator/package.json +++ b/windmill-yaml-validator/package.json @@ -1,6 +1,6 @@ { "name": "windmill-yaml-validator", - "version": "1.804.0", + "version": "1.805.0", "description": "YAML validator for Windmill flow, schedule, and trigger files", "main": "dist/index.js", "types": "dist/index.d.ts", From 15c2b81d6c182bbe63c2756b600a2d24eb194b0e Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 8 Sep 2026 06:19:32 +0000 Subject: [PATCH 016/144] chore: run local codex review on gpt-6-astra, bump codex cli pin (#11011) * chore: run local codex review on gpt-6-astra and bump codex cli pin Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01X8u6o9MRAs2Rz16UbKQaD9 * fix: keep local codex review alive when --version is unparseable Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01X8u6o9MRAs2Rz16UbKQaD9 --------- Co-authored-by: Claude Opus 5 (1M context) --- .agents/skills/local-review-codex/SKILL.md | 7 ++--- .agents/skills/local-review-codex/run.sh | 31 +++++++++++++++++++--- .github/workflows/codex-pr-review.yml | 2 +- AGENTS.md | 2 +- 4 files changed, 33 insertions(+), 9 deletions(-) diff --git a/.agents/skills/local-review-codex/SKILL.md b/.agents/skills/local-review-codex/SKILL.md index cc932f7a4b..ec1277fe3a 100644 --- a/.agents/skills/local-review-codex/SKILL.md +++ b/.agents/skills/local-review-codex/SKILL.md @@ -1,6 +1,6 @@ --- name: local-review-codex -description: Run the CI Codex PR review locally against this branch's unpushed work (committed + uncommitted) before pushing. Same policy, model, and reasoning effort as the codex-pr-review GitHub action. +description: Run the CI Codex PR review locally against this branch's unpushed work (committed + uncommitted) before pushing. Same policy and reasoning effort as the codex-pr-review GitHub action, on a newer model. --- # Local Codex Review (pre-push) @@ -11,17 +11,18 @@ before the PR exists. Use this before `git push` on a non-trivial change. **Correspondence with CI** — identical: - Policy: `REVIEW.md` (severity triage, public-surface checklist, AGENTS.md compliance, test coverage). -- Model: `gpt-5.6-sol`, `model_reasoning_effort="xhigh"`. +- Reasoning effort: `model_reasoning_effort="xhigh"`. - Output: markdown starting with `## Codex Review`, findings tagged P0 / P1 / P2 with file:line. **Differences from CI** — local-only: +- Model is `gpt-6-astra`; CI stays on `gpt-5.6-sol`. Not an oversight to reconcile: `gpt-6-astra` is confirmed on the ChatGPT auth `codex login` uses locally, while CI authenticates with `OPENAI_API_KEY` (`codex-pr-review.yml` prefers it over `CODEX_AUTH_JSON`) and that tier is unverified for the model. Move CI once API access is confirmed, or once CI switches to `CODEX_AUTH_JSON`. - Scope is the current branch vs `main` at the merge-base, **including uncommitted changes** (CI reviews a pushed PR diff). - Sandbox is `read-only` (CI uses `danger-full-access` on an ephemeral runner). Codex reads the diff and files but cannot modify your working tree. - Fresh context is inherent: `codex exec` is a separate cold process, so it does not anchor on the current chat session — the same reason `local-review` insists on a subagent. ## Prerequisites -- `codex` CLI **>= 0.144.1** installed and authed (`codex login` or `OPENAI_API_KEY`). Older CLIs reject `gpt-5.6-sol` with "requires a newer version of Codex". Upgrade with `npm install --global @openai/codex@0.144.1` (may need `sudo` for a global install). Keep this in sync with the pin in `.github/workflows/codex-pr-review.yml`. +- `codex` CLI **>= 0.153.4** installed and authed via `codex login` (an `OPENAI_API_KEY` in the environment takes priority and may not reach `gpt-6-astra` — see the model note above). Older CLIs reject the model with "requires a newer version of Codex"; `run.sh` checks the version up front. Upgrade with `npm install --global @openai/codex@0.153.4` (may need `sudo` for a global install). This matches the pin in `.github/workflows/codex-pr-review.yml` — the CLI version is the same on both sides, only the model differs. - `git fetch` the base ref if it's stale, so the merge-base is accurate. ## Run diff --git a/.agents/skills/local-review-codex/run.sh b/.agents/skills/local-review-codex/run.sh index d6491099c2..948d3820eb 100755 --- a/.agents/skills/local-review-codex/run.sh +++ b/.agents/skills/local-review-codex/run.sh @@ -1,21 +1,44 @@ #!/usr/bin/env bash # Local Codex review — mirrors the .github/workflows/codex-pr-review.yml CI job, # but scoped to this branch's unpushed work (committed + uncommitted) so you can -# review before pushing. Same policy (REVIEW.md), same model (gpt-5.6-sol) and -# reasoning effort (xhigh) as CI. Runs read-only: Codex cannot modify your tree. +# review before pushing. Same policy (REVIEW.md) and reasoning effort (xhigh) as CI. +# +# The model deliberately differs from CI: gpt-6-astra is confirmed available on the +# ChatGPT auth `codex login` uses here, but CI authenticates with OPENAI_API_KEY and +# that tier is unverified for it, so codex-pr-review.yml stays on gpt-5.6-sol. # # Usage: run.sh [BASE_REF] (BASE_REF defaults to "main") set -euo pipefail +MODEL="gpt-6-astra" +CODEX_MIN="0.153.4" + BASE_REF="${1:-main}" REPO_ROOT="$(git rev-parse --show-toplevel)" cd "$REPO_ROOT" if ! command -v codex >/dev/null 2>&1; then - echo "codex CLI not found. Install with: npm install --global @openai/codex@0.144.1" >&2 + echo "codex CLI not found. Install with: npm install --global @openai/codex@$CODEX_MIN" >&2 exit 1 fi +# Older CLIs reject the model with an error that never names the CLI version as the +# cause, so check it up front rather than letting the exec fail opaquely. The `|| true` +# keeps an unrecognised --version format from aborting under `set -e`: an unparseable +# version means "cannot tell", which must fall through to the exec, not kill the review. +CODEX_VER="$(codex --version 2>/dev/null | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1 || true)" +if [ -n "$CODEX_VER" ] && [ "$(printf '%s\n%s\n' "$CODEX_MIN" "$CODEX_VER" | sort -V | head -1)" != "$CODEX_MIN" ]; then + echo "codex $CODEX_VER is too old for $MODEL (need >= $CODEX_MIN). Upgrade with: npm install --global @openai/codex@$CODEX_MIN" >&2 + exit 1 +fi + +# codex prefers OPENAI_API_KEY over the ChatGPT credentials `codex login` stores, and +# that tier is not confirmed for $MODEL — the resulting failure names the model, not the +# auth that selected it. +if [ -n "${OPENAI_API_KEY:-}" ]; then + echo "warning: OPENAI_API_KEY is set and takes priority over 'codex login' credentials; $MODEL may be unavailable on that tier." >&2 +fi + # Resolve the base to a concrete commit, preferring a local ref but falling back to # the remote-tracking ref — checkouts (CI, single-branch clones) often have only # origin/main, not a local main. @@ -80,7 +103,7 @@ EOF codex exec \ -C "$REPO_ROOT" \ - -m gpt-5.6-sol \ + -m "$MODEL" \ -c 'model_reasoning_effort="xhigh"' \ -s read-only \ -o "$OUT" \ diff --git a/.github/workflows/codex-pr-review.yml b/.github/workflows/codex-pr-review.yml index 26b2d9aae8..7a3882df7c 100644 --- a/.github/workflows/codex-pr-review.yml +++ b/.github/workflows/codex-pr-review.yml @@ -219,7 +219,7 @@ jobs: - name: Install Codex CLI if: steps.codex_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true' - run: npm install --global @openai/codex@0.144.1 + run: npm install --global @openai/codex@0.153.4 - name: Configure Codex auth if: steps.codex_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true' diff --git a/AGENTS.md b/AGENTS.md index 71c59b479d..2b204a65db 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -36,7 +36,7 @@ Open-source platform for internal tools, workflows, API integrations, background - **Backend patterns**: use the `rust-backend` skill when writing Rust code - **Frontend patterns**: use the `svelte-frontend` skill when writing Svelte code. Do NOT edit svelte files unless you have read that skill. - **Frontend UUIDs**: do not call `crypto.randomUUID()` in frontend code. Import `randomUUID` from `$lib/utils/uuid` instead. -- **Code review**: review the current PR or branch against the shared review policy in `REVIEW.md` (severity triage, public-surface checklist, AGENTS.md compliance, test-coverage assessment). The skill at `.agents/skills/local-review/SKILL.md` orchestrates it. All three CLIs auto-discover the same SKILL — Claude reads `.claude/skills/` (symlinked to the canonical `.agents/skills/` file), Codex and Pi read `.agents/skills/` directly. Invoke with `/local-review` in Claude Code, `$local-review` (or `/skills` selector) in Codex, or `pi --skill local-review` / `/skill:local-review` in Pi. For a Codex-driven pass that mirrors the `codex-pr-review` GitHub action against your unpushed work (committed + uncommitted) before you push, use `/local-review-codex` (`.agents/skills/local-review-codex/`) — same `REVIEW.md` policy, `gpt-5.6-sol`, `xhigh` reasoning; requires the `codex` CLI >= 0.144.1. +- **Code review**: review the current PR or branch against the shared review policy in `REVIEW.md` (severity triage, public-surface checklist, AGENTS.md compliance, test-coverage assessment). The skill at `.agents/skills/local-review/SKILL.md` orchestrates it. All three CLIs auto-discover the same SKILL — Claude reads `.claude/skills/` (symlinked to the canonical `.agents/skills/` file), Codex and Pi read `.agents/skills/` directly. Invoke with `/local-review` in Claude Code, `$local-review` (or `/skills` selector) in Codex, or `pi --skill local-review` / `/skill:local-review` in Pi. For a Codex-driven pass that mirrors the `codex-pr-review` GitHub action against your unpushed work (committed + uncommitted) before you push, use `/local-review-codex` (`.agents/skills/local-review-codex/`) — same `REVIEW.md` policy and `xhigh` reasoning, on `gpt-6-astra` rather than the action's `gpt-5.6-sol`; requires the `codex` CLI >= 0.153.4. - **Domain guides**: `.claude/skills/native-trigger/` and `frontend/tutorial-system-guide.mdc` - **Brand/UI guidelines**: `frontend/brand-guidelines.md` - **Domain vocabulary**: `CONTEXT.md` — the words this codebase uses for its own concepts (step, step setting, trigger step, …). Name things the way it does. From 621fac55abcd1859e8c8c06e5f4412e61bb85d59 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 8 Sep 2026 06:32:30 +0000 Subject: [PATCH 017/144] feat: durable dbt state per environment, and `--defer` onto it (#10975) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: durable dbt state per environment, and `--defer` onto it `dbt retry` worked off two artifacts and only one was durable: `dbt_run_state` holds `run_results.json` keyed by principal, and the manifest lived on worker-local disk under a four-generation cache. That is enough to resume the last run and nothing else — the next run of a project usually lands on a worker holding neither artifact — so deferral had nothing to read. Adds `dbt_environment_state`: one row per (workspace, script path, environment), holding `manifest.json` and `run_results.json` from the last successful run, with the blob inline under `DBT_STATE_INLINE_MAX_BYTES` and in the workspace's object storage above it. Environment is the warehouse, the target, and the database and schema they resolve to, so a repointed warehouse or a moved schema reads as an environment nothing has published rather than as state whose relation names no longer fit. A run publishes it when its graph becomes what the script owns and it succeeded — the same condition, and the same reason: an invocation that scoped its own model set describes where the caller put those relations, not where the project's models live. `defer` is a `build` command-block field defaulting to the descriptor's own, and the state is materialised into the job directory for `--defer --state`. The retry path already did that materialisation for `dbt retry`; both go through one `write_state_dir` now. `--state` is also where `dbt retry` reads the run it resumes, so a retry on dbt-core 1.x takes `--defer-state` instead, and one on an engine without that flag is refused before the build rather than rebuilding its nodes with every unbuilt `ref()` resolving into the schema this run writes into. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Ned2pmRJwB3GpenEcrA9TF * fix: address the local review of the dbt environment state The oversized-artifact home moves from the workspace's object storage to the instance's, where every other internal worker artifact already lives. The workspace bucket is the one members read and write through `job_helpers/*` with a caller-supplied key and only `volumes/` is reserved there, so a manifest under it is one any member could replace — and the next deferring run would hand dbt an attacker-chosen `defer_relation` for every unbuilt `ref()` while holding the script's warehouse credentials. The environment key takes the target dbt actually runs rather than the descriptor's `profile.target`, which is absent whenever the target is inherited from the workspace warehouse or the project's own `profiles.yml` — filing every inherited target under one empty name, while a `target.name` macro decides where a model is built. `write_profiles` returns a named struct now that it resolves one more thing. Publishing takes the row's lock before uploading, so two publishers of one environment cannot interleave their uploads and leave one run's manifest beside another's results, and carries the live-dbt-script guard the retry state already had, so a job finishing after its script was renamed, archived or deleted cannot recreate state at a path for whatever is created there next. A rename now clears the environment state instead of moving it: an oversized artifact's key is derived from the path, so a moved row would keep pointing at a key a script created at the old path publishes over. A build recovered by the automatic in-job node retry publishes its manifest without results — `run_results.json` is then the retry's, naming only the nodes it redid — and the refusal for an environment with nothing published names the runs that cannot publish rather than suggesting a run that would not help. Co-Authored-By: Claude Opus 5 (1M context) * fix: serialize dbt state publishers on an advisory lock The row lock only serializes publishers once a row exists, and the first publish of an environment — two runs of a newly deployed script — is exactly when two of them are most likely to race and interleave their uploads. Co-Authored-By: Claude Opus 5 (1M context) * fix: make dbt state publication atomic and bind it to the version that ran Every publication now writes its own object keys and the row switches to them in one statement, so an upload never overwrites an artifact the committed row still names: a run failing between its two uploads, or between them and its row, leaves the state pointing at the pair it already had. The objects a commit displaces are dropped afterwards — never before, since a reader that has already read the row is about to fetch them — and a reader that loses that race re-reads the row once rather than reporting a state that is there. What a publication uploaded and then could not commit is dropped on the way out. The write's guard names the VERSION rather than the path: the live dbt script there must be the one this job ran, or a later version of it. "Some live dbt script is here" is also satisfied by a script created at a path this one was renamed away from, and this job's manifest would then become that project's deferral state. A preview names no version and so publishes nothing. A `show` defers too. It compiles the model it previews, so a model whose upstream this environment built and this run did not is exactly the case a deferral exists for, and every engine takes the flags on it. Three comments said "the workspace's object storage" where the code deliberately uses the instance's, which is the whole security argument; `mib()` labelled MiB values MB. Co-Authored-By: Claude Opus 5 (1M context) * fix: hold the script row across a dbt state publication, and let a rename move it The version guard read `script` without a lock, so lifecycle cleanup could find no environment row to clear, finish, and leave this transaction to commit state at a path a new script goes on to occupy. It now holds that row (`FOR SHARE`) for the rest of the publication — taken before the sidecar, the order every other dbt writer takes — and the artifacts are uploaded before the transaction, so the lock covers the row work rather than a network round trip. A commit that reports an error may still have committed: what was lost can be the acknowledgement. Dropping this run's objects then leaves the committed row naming objects that are gone, so an orphan is the cheaper side to take. A failed second upload left the manifest it had already written behind; it is dropped now. Per-publication keys retired the reason a rename cleared the environment state rather than moving it: the path is only a prefix, and the row is what names an artifact, so a script created at the old path can no longer publish over a moved row. The rename moves both halves again. `dbt ls` gets the deferral flags too, without which a `result:` selector — which reads `run_results.json` out of the state directory, and which `select` passes to dbt verbatim — fails before the build that would have honoured it. Also: the migration was the last site describing the workspace's object storage rather than the instance's, `publication_lock` folded 32 bits where it claimed 64, and `ResolvedProfile` had taken `write_profiles`'s doc block. Co-Authored-By: Claude Opus 5 (1M context) * fix: a deferring dbt run never publishes the state it read `publishes_ownership` reads the CALLER's overrides, so a descriptor that already narrows `select` needs none and a run of it with `defer: true` published. A deferring run built some of the relations its manifest names and resolved the rest out of the state it read, so recording that manifest claims relations nothing built — and a model renamed since is recorded under a name only a full build creates, breaking every later deferral until one repairs it. Also: `publication_lock` parsed 16 hex digits as `i64`, which overflows for every digest with the top bit set — half of them — collapsing those environments onto one advisory key; and a failure to open the transaction returned without dropping the objects already uploaded. Co-Authored-By: Claude Opus 5 (1M context) * fix: only a deployed dbt run publishes state, and key its objects per execution A preview carries a caller-supplied `script_hash` into `runnable_id` (`run_preview_script`), so the version guard alone let anyone who may run a job publish arbitrary content as a deployed script's deferral state. The job's KIND is checked beside it now. Verified: a preview submitted with the deployed path and hash builds and leaves the row untouched. Object keys carry a per-execution nonce. Zombie recovery re-runs a job under its own id, so keyed on that alone a second attempt overwrote the objects the first attempt's committed row still named, then read those same keys back as displaced and dropped them — leaving the row unreadable. The displaced set is also filtered against this publication's own keys, so the invariant is stated rather than re-derived from the key format. A project-owned `profiles.yml` that templates its schema or database is refused a deferral: dbt renders those and Windmill does not, so two renderings resolve to one `relation_root` and would share one environment key. Plainly absent is left alone — that is the adapter's default, which does not move. The deferral log line now says the run publishes no state of its own, which was otherwise invisible. Co-Authored-By: Claude Opus 5 (1M context) * fix: a templated profile location publishes no dbt state either, on every path A `dbt_profile` resource is one block of the user's own `profiles.yml` copied through unchanged, and `profile.schema` is written as given, so either can carry a template dbt renders and this runtime does not — exactly as a project-owned file can. Only the project-owned path detected it. And the refusal now covers publication as well as deferral: a published template would sit under a key a literal profile shares, so de-templating later would make that stale manifest readable as the new location's. Co-Authored-By: Claude Opus 5 (1M context) * fix: recognise Jinja statement blocks as a rendered dbt profile location dbt renders a profile through Jinja, so `{% if env_var('ENV') == 'prod' %}…{% endif %}` moves a schema exactly as an `env_var()` substitution does — and only `{{` was detected, so such a profile published and deferred under one environment key for every rendering. One predicate now serves both profile paths, with a test for each delimiter. Co-Authored-By: Claude Opus 5 (1M context) * fix: a dbt state read outruns successive publications rather than one The loader re-read once, which answers a single publication overtaking it: a reader takes no lock and the advisory lock is released before the displaced objects are dropped, so back-to-back publications could each overtake the same read and the second was reported as a missing object. It now re-reads for as long as the row keeps MOVING, bounded, and reports only when an unmoved row's objects are genuinely gone. Co-Authored-By: Claude Opus 5 (1M context) * docs: a dbt state read outruns successive publications, not one Co-Authored-By: Claude Opus 5 (1M context) * docs: name both ways a dbt state read can fail Co-Authored-By: Claude Opus 5 (1M context) * fix: length-prefix the dbt environment key's components A dbt target name and a schema are both the user's own strings, so joining them on `|` let one component spell another tuple's key: `prod|analytics` + `scratch` and `prod` + `analytics|scratch` were one environment, and a profile moving between them read as the same one rather than as one nothing has published — the collision the key exists to prevent. The schema and database are also taken apart now rather than through `relation_root`'s own join, so neither can absorb the other's delimiter. Co-Authored-By: Claude Opus 5 (1M context) * fix: name the dbt environment in words where a message shows it The key is length-prefixed for storage, which is not something to put in front of a caller: the "nothing published yet" refusal now reads "warehouse `main`, target `prod`, relations in `dbt_wh_defer.analytics`". The worked example of the encoding also miscounted a component. Co-Authored-By: Claude Opus 5 (1M context) * fix: delete a script version in the transaction that cleans up after it `delete_script_by_hash` soft-deleted through the pool, committing before the cleanup that follows it in `tx`. In that window the path has no live version, so a concurrent deploy can take it — and `clear_dbt_script_state_if_path_retired` then finds that new script live, keeps the deleted project's dbt state, and leaves the replacement able to defer through its manifest. The update moves into the same transaction, which is what `archive_script_by_hash` beside it already does. The retirement guard itself was pinned by nothing: the existing test moved the only row away before calling the conditional clear, so it could not fail. `state_goes_only_once_no_live_version_is_left` covers both directions — a second live version keeps the state, the last one leaving takes it — and fails if the predicate is inverted. Co-Authored-By: Claude Opus 5 (1M context) * fix: archive a script by path in the transaction that cleans up after it The last of the four routes still writing outside its own cleanup transaction. Archived on its own, a cleanup that then fails leaves dbt state at a path no live version occupies, and whatever is created there next can defer through it. The by-hash archive and both deletes already take their write in `tx`; this makes the set uniform. Two comments beside those clears still called the state the RETRY state alone, which the rename made false — they cover both halves now — and the merged verification list had two `11.`, main's #10978 having inserted an item above it. Co-Authored-By: Claude Opus 5 (1M context) * feat: refuse a dbt state selector the engines resolve inconsistently `state:`, `result:` and `source_status:` selectors resolve against the artifacts in `--state`, which only a deferring run is handed. The engines disagree about what happens without one, and two of the three disagree silently: dbt-core 1.x raises, but dbt-sa-cli 2.x and fusion read a missing state as an empty one and exit 0, so `state:modified` builds nothing and `state:new` builds the whole project, each reporting success. Refuse them up front instead, naming `defer`. From the descriptor they are refused outright, since that selection also decides which nodes the script owns and the deploy resolves it with no state at all. `source_status:` is refused under any setting: it compares `sources.json`, which no run publishes here. A caller's selection is now allowed to match nothing, which is what `state:modified+` returns when nothing changed since the published state. It is stored as that run's own snapshot and never becomes what the script owns, so the ownership-wipe the refusal guarded against cannot happen. The descriptor's selection still may not. Co-Authored-By: Claude Opus 5 (1M context) * fix: refuse a dbt result selector the published state cannot answer Round 18 findings. Codex P1: `defer` alone was enough to allow a `result:` selector, but a build recovered by node retry publishes a manifest with no `run_results.json` — the only file such a selector reads. dbt-core then raises an internal error and the Rust engines match nothing and exit 0. The deferral now reports whether the state carries results, and a `result:` selection against one that does not is refused, naming the run that published it. Claude P2: a `parse` returns before `defer` is read, so its deferral is always absent and "turn `defer` on" was advice that led nowhere. The check now distinguishes a run that could defer from a command that never does, and the parse path says so. Codex P2 / Claude P2: the roadmap still listed `state:modified` as out of scope while the same file documented it as working. Narrowed both that line and the scope list to the slim-CI work that genuinely remains. Also pins the invariant the relaxed empty-selection guard rests on: an overridden selection must not publish ownership, or an empty caller selection would wipe the script's graph. Co-Authored-By: Claude Opus 5 (1M context) * fix: exempt an empty dbt selection by method, not by who chose it Round 19 findings. Codex P1: the empty-selection exemption keyed on whether the caller overrode the selection, so a misspelled model name resolved to nothing, passed the guard and reported a build that did its work. Key it on the selector instead: only a `state:` or `result:` method may match nothing, its empty answer being a real one. Every other selection matching nothing is refused again, from a run as from the descriptor, each with the message that applies to it. Claude P2: the spec still described a node-retry-recovered publication as one where `result:` selectors merely lose their input, which the previous commit stopped being true, and the section stating the selector rules recorded neither the `result:`-without-results refusal nor the `parse` one. Both written down. Also drops the refusal's claim that the publishing run WAS recovered by node retry: an unreadable file reaches the same absent-results state, and the remedy is the same either way. Co-Authored-By: Claude Opus 5 (1M context) * docs: record why an exempted empty dbt selection cannot wipe the graph The safety argument left with the origin-based condition it justified. Under the method-based one it is a consequence of the descriptor refusal in check_state_selectors, two hops from this site, so state it here: relaxing that refusal would let a descriptor-narrowed `state:modified+` reach the exemption and be ingested as owning nothing. Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Opus 5 (1M context) --- ...bce43c749a44cbc4e7b8f719bf2a598ce57f3.json | 21 + ...5cc73eef1e3d01825031ba263881b7cfd5ed4.json | 14 + ...938eb135d6eb9714ca115359221d7c02861f0.json | 17 + ...beeedc63b4a941e5b17ab7bb5d3c259f05147.json | 24 + ...9b6a36c50d70e20c5a10310f75a39930db521.json | 16 + ...e69ef1e0784e9f25ada27e1b0f5cc0944e9ba.json | 16 + ...7d192f5e926b932c3dd0e1bef4822cf722630.json | 48 ++ ...dcc9d963cc033582bf2e945e8bf3a301b4247.json | 22 + ...19b2c2b7df2788c9eb0115abc60ea7b733d64.json | 15 + ...9eb8e43d384084b68dc1bbb3b961206ff4b6c.json | 15 + ...205427cc50af031a84d334f65f870d86302ac.json | 48 ++ ...821dde2c093bb926b77ec3f99a1c0d9283a14.json | 23 + ...01cca0bcbc832f01eb3340050504b8b3875a2.json | 30 + ...d313db5945faa0b93c127833e44d38665a392.json | 30 + ...60904135713_dbt_environment_state.down.sql | 1 + ...0260904135713_dbt_environment_state.up.sql | 52 ++ .../parsers/windmill-parser-yaml/src/dbt.rs | 70 +- backend/summarized_schema.txt | 2 + backend/windmill-api-scripts/src/scripts.rs | 59 +- backend/windmill-common/src/dbt_manifest.rs | 93 ++- .../tests/dbt_graph_storage.rs | 185 ++++- backend/windmill-worker/src/dbt_executor.rs | 705 +++++++++++++++-- backend/windmill-worker/src/dbt_state.rs | 748 ++++++++++++++++++ backend/windmill-worker/src/lib.rs | 1 + docs/dbt-runtime.md | 362 ++++++++- frontend/src/lib/script_helpers.ts | 5 + 26 files changed, 2485 insertions(+), 137 deletions(-) create mode 100644 backend/.sqlx/query-3fc12582cbae2ebc77ecfcaeed4bce43c749a44cbc4e7b8f719bf2a598ce57f3.json create mode 100644 backend/.sqlx/query-446909206f289fc3f9201a6ad025cc73eef1e3d01825031ba263881b7cfd5ed4.json create mode 100644 backend/.sqlx/query-4ce90ff818e5058a7e31da24abd938eb135d6eb9714ca115359221d7c02861f0.json create mode 100644 backend/.sqlx/query-58ec340c78af046a40296b15543beeedc63b4a941e5b17ab7bb5d3c259f05147.json create mode 100644 backend/.sqlx/query-5c7260785ebcece2ddb04dc85d09b6a36c50d70e20c5a10310f75a39930db521.json create mode 100644 backend/.sqlx/query-7f421bdf3dc4d47b36457af703ce69ef1e0784e9f25ada27e1b0f5cc0944e9ba.json create mode 100644 backend/.sqlx/query-8165a447f458d62f7bafb9827d87d192f5e926b932c3dd0e1bef4822cf722630.json create mode 100644 backend/.sqlx/query-a06e1d9f6f95e4c4c2b98310ebddcc9d963cc033582bf2e945e8bf3a301b4247.json create mode 100644 backend/.sqlx/query-b820bf7b0a93476fb7762e8ff2819b2c2b7df2788c9eb0115abc60ea7b733d64.json create mode 100644 backend/.sqlx/query-d2c02f1c7f4555fa849f5cd86169eb8e43d384084b68dc1bbb3b961206ff4b6c.json create mode 100644 backend/.sqlx/query-d8b7241518ce0822288fb123b48205427cc50af031a84d334f65f870d86302ac.json create mode 100644 backend/.sqlx/query-f5e061eed38d483980ee3691aea821dde2c093bb926b77ec3f99a1c0d9283a14.json create mode 100644 backend/.sqlx/query-f963ea31d1744ff0d76ff86140f01cca0bcbc832f01eb3340050504b8b3875a2.json create mode 100644 backend/.sqlx/query-ffde6e45841090440bc99e22788d313db5945faa0b93c127833e44d38665a392.json create mode 100644 backend/migrations/20260904135713_dbt_environment_state.down.sql create mode 100644 backend/migrations/20260904135713_dbt_environment_state.up.sql create mode 100644 backend/windmill-worker/src/dbt_state.rs diff --git a/backend/.sqlx/query-3fc12582cbae2ebc77ecfcaeed4bce43c749a44cbc4e7b8f719bf2a598ce57f3.json b/backend/.sqlx/query-3fc12582cbae2ebc77ecfcaeed4bce43c749a44cbc4e7b8f719bf2a598ce57f3.json new file mode 100644 index 0000000000..1268656e34 --- /dev/null +++ b/backend/.sqlx/query-3fc12582cbae2ebc77ecfcaeed4bce43c749a44cbc4e7b8f719bf2a598ce57f3.json @@ -0,0 +1,21 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_environment_state (workspace_id, script_path, environment, job_id,\n manifest, manifest_key, run_results,\n run_results_key, updated_at)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, now())\n ON CONFLICT (workspace_id, script_path, environment) DO UPDATE SET\n job_id = EXCLUDED.job_id, manifest = EXCLUDED.manifest,\n manifest_key = EXCLUDED.manifest_key, run_results = EXCLUDED.run_results,\n run_results_key = EXCLUDED.run_results_key, updated_at = now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Text", + "Uuid", + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "3fc12582cbae2ebc77ecfcaeed4bce43c749a44cbc4e7b8f719bf2a598ce57f3" +} diff --git a/backend/.sqlx/query-446909206f289fc3f9201a6ad025cc73eef1e3d01825031ba263881b7cfd5ed4.json b/backend/.sqlx/query-446909206f289fc3f9201a6ad025cc73eef1e3d01825031ba263881b7cfd5ed4.json new file mode 100644 index 0000000000..7f950346ee --- /dev/null +++ b/backend/.sqlx/query-446909206f289fc3f9201a6ad025cc73eef1e3d01825031ba263881b7cfd5ed4.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM script WHERE workspace_id = $1 AND hash = 1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "446909206f289fc3f9201a6ad025cc73eef1e3d01825031ba263881b7cfd5ed4" +} diff --git a/backend/.sqlx/query-4ce90ff818e5058a7e31da24abd938eb135d6eb9714ca115359221d7c02861f0.json b/backend/.sqlx/query-4ce90ff818e5058a7e31da24abd938eb135d6eb9714ca115359221d7c02861f0.json new file mode 100644 index 0000000000..913c16c31d --- /dev/null +++ b/backend/.sqlx/query-4ce90ff818e5058a7e31da24abd938eb135d6eb9714ca115359221d7c02861f0.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_environment_state (workspace_id, script_path, environment, job_id,\n manifest)\n SELECT $1::varchar, $2::varchar, 'main||analytics|wh'::text, $3::uuid, '{}'::text\n WHERE EXISTS (SELECT 1 FROM script\n WHERE workspace_id = $1 AND path = $2\n AND deleted = false AND archived = false\n AND language = 'dbt'\n AND (hash = $4 OR $4 = ANY(parent_hashes)))\n ON CONFLICT (workspace_id, script_path, environment) DO UPDATE SET job_id = EXCLUDED.job_id", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Uuid", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "4ce90ff818e5058a7e31da24abd938eb135d6eb9714ca115359221d7c02861f0" +} diff --git a/backend/.sqlx/query-58ec340c78af046a40296b15543beeedc63b4a941e5b17ab7bb5d3c259f05147.json b/backend/.sqlx/query-58ec340c78af046a40296b15543beeedc63b4a941e5b17ab7bb5d3c259f05147.json new file mode 100644 index 0000000000..b2a0d728cd --- /dev/null +++ b/backend/.sqlx/query-58ec340c78af046a40296b15543beeedc63b4a941e5b17ab7bb5d3c259f05147.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT 1 FROM script\n WHERE workspace_id = $1 AND path = $2\n AND deleted = false AND archived = false AND language = 'dbt'\n AND (hash = $3 OR $3 = ANY(parent_hashes))\n FOR SHARE", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8" + ] + }, + "nullable": [ + null + ] + }, + "hash": "58ec340c78af046a40296b15543beeedc63b4a941e5b17ab7bb5d3c259f05147" +} diff --git a/backend/.sqlx/query-5c7260785ebcece2ddb04dc85d09b6a36c50d70e20c5a10310f75a39930db521.json b/backend/.sqlx/query-5c7260785ebcece2ddb04dc85d09b6a36c50d70e20c5a10310f75a39930db521.json new file mode 100644 index 0000000000..09efdc7340 --- /dev/null +++ b/backend/.sqlx/query-5c7260785ebcece2ddb04dc85d09b6a36c50d70e20c5a10310f75a39930db521.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_environment_state (workspace_id, script_path, environment, job_id,\n manifest)\n VALUES ($1, $2, 'main||analytics|wh', $3, '{}')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "5c7260785ebcece2ddb04dc85d09b6a36c50d70e20c5a10310f75a39930db521" +} diff --git a/backend/.sqlx/query-7f421bdf3dc4d47b36457af703ce69ef1e0784e9f25ada27e1b0f5cc0944e9ba.json b/backend/.sqlx/query-7f421bdf3dc4d47b36457af703ce69ef1e0784e9f25ada27e1b0f5cc0944e9ba.json new file mode 100644 index 0000000000..c5740cfb88 --- /dev/null +++ b/backend/.sqlx/query-7f421bdf3dc4d47b36457af703ce69ef1e0784e9f25ada27e1b0f5cc0944e9ba.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE dbt_environment_state SET script_path = $3\n WHERE workspace_id = $1 AND script_path = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "7f421bdf3dc4d47b36457af703ce69ef1e0784e9f25ada27e1b0f5cc0944e9ba" +} diff --git a/backend/.sqlx/query-8165a447f458d62f7bafb9827d87d192f5e926b932c3dd0e1bef4822cf722630.json b/backend/.sqlx/query-8165a447f458d62f7bafb9827d87d192f5e926b932c3dd0e1bef4822cf722630.json new file mode 100644 index 0000000000..f4bd63c718 --- /dev/null +++ b/backend/.sqlx/query-8165a447f458d62f7bafb9827d87d192f5e926b932c3dd0e1bef4822cf722630.json @@ -0,0 +1,48 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT job_id, manifest, manifest_key, run_results, run_results_key\n FROM dbt_environment_state\n WHERE workspace_id = $1 AND script_path = $2 AND environment = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "job_id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "manifest", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "manifest_key", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "run_results", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "run_results_key", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + false, + true, + true, + true, + true + ] + }, + "hash": "8165a447f458d62f7bafb9827d87d192f5e926b932c3dd0e1bef4822cf722630" +} diff --git a/backend/.sqlx/query-a06e1d9f6f95e4c4c2b98310ebddcc9d963cc033582bf2e945e8bf3a301b4247.json b/backend/.sqlx/query-a06e1d9f6f95e4c4c2b98310ebddcc9d963cc033582bf2e945e8bf3a301b4247.json new file mode 100644 index 0000000000..909e6ad42d --- /dev/null +++ b/backend/.sqlx/query-a06e1d9f6f95e4c4c2b98310ebddcc9d963cc033582bf2e945e8bf3a301b4247.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT pg_advisory_xact_lock($1)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "pg_advisory_xact_lock", + "type_info": "Void" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + null + ] + }, + "hash": "a06e1d9f6f95e4c4c2b98310ebddcc9d963cc033582bf2e945e8bf3a301b4247" +} diff --git a/backend/.sqlx/query-b820bf7b0a93476fb7762e8ff2819b2c2b7df2788c9eb0115abc60ea7b733d64.json b/backend/.sqlx/query-b820bf7b0a93476fb7762e8ff2819b2c2b7df2788c9eb0115abc60ea7b733d64.json new file mode 100644 index 0000000000..7476ae9871 --- /dev/null +++ b/backend/.sqlx/query-b820bf7b0a93476fb7762e8ff2819b2c2b7df2788c9eb0115abc60ea7b733d64.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM dbt_environment_state WHERE workspace_id = $1 AND script_path = $2\n AND NOT EXISTS (SELECT 1 FROM script\n WHERE workspace_id = $1 AND path = $2\n AND deleted = false AND archived = false)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "b820bf7b0a93476fb7762e8ff2819b2c2b7df2788c9eb0115abc60ea7b733d64" +} diff --git a/backend/.sqlx/query-d2c02f1c7f4555fa849f5cd86169eb8e43d384084b68dc1bbb3b961206ff4b6c.json b/backend/.sqlx/query-d2c02f1c7f4555fa849f5cd86169eb8e43d384084b68dc1bbb3b961206ff4b6c.json new file mode 100644 index 0000000000..9c75f7ac97 --- /dev/null +++ b/backend/.sqlx/query-d2c02f1c7f4555fa849f5cd86169eb8e43d384084b68dc1bbb3b961206ff4b6c.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM dbt_environment_state WHERE workspace_id = $1 AND script_path = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "d2c02f1c7f4555fa849f5cd86169eb8e43d384084b68dc1bbb3b961206ff4b6c" +} diff --git a/backend/.sqlx/query-d8b7241518ce0822288fb123b48205427cc50af031a84d334f65f870d86302ac.json b/backend/.sqlx/query-d8b7241518ce0822288fb123b48205427cc50af031a84d334f65f870d86302ac.json new file mode 100644 index 0000000000..748ca6d9ec --- /dev/null +++ b/backend/.sqlx/query-d8b7241518ce0822288fb123b48205427cc50af031a84d334f65f870d86302ac.json @@ -0,0 +1,48 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT job_id, manifest, manifest_key, run_results, run_results_key\n FROM dbt_environment_state\n WHERE workspace_id = $1 AND script_path = $2 AND environment = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "job_id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "manifest", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "manifest_key", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "run_results", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "run_results_key", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + false, + true, + true, + true, + true + ] + }, + "hash": "d8b7241518ce0822288fb123b48205427cc50af031a84d334f65f870d86302ac" +} diff --git a/backend/.sqlx/query-f5e061eed38d483980ee3691aea821dde2c093bb926b77ec3f99a1c0d9283a14.json b/backend/.sqlx/query-f5e061eed38d483980ee3691aea821dde2c093bb926b77ec3f99a1c0d9283a14.json new file mode 100644 index 0000000000..1c79a53ab0 --- /dev/null +++ b/backend/.sqlx/query-f5e061eed38d483980ee3691aea821dde2c093bb926b77ec3f99a1c0d9283a14.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT count(*) FROM dbt_environment_state WHERE workspace_id = $1 AND script_path = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "f5e061eed38d483980ee3691aea821dde2c093bb926b77ec3f99a1c0d9283a14" +} diff --git a/backend/.sqlx/query-f963ea31d1744ff0d76ff86140f01cca0bcbc832f01eb3340050504b8b3875a2.json b/backend/.sqlx/query-f963ea31d1744ff0d76ff86140f01cca0bcbc832f01eb3340050504b8b3875a2.json new file mode 100644 index 0000000000..540d2e0e08 --- /dev/null +++ b/backend/.sqlx/query-f963ea31d1744ff0d76ff86140f01cca0bcbc832f01eb3340050504b8b3875a2.json @@ -0,0 +1,30 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT manifest_key, run_results_key FROM dbt_environment_state\n WHERE workspace_id = $1 AND script_path = $2 AND environment = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "manifest_key", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "run_results_key", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + true, + true + ] + }, + "hash": "f963ea31d1744ff0d76ff86140f01cca0bcbc832f01eb3340050504b8b3875a2" +} diff --git a/backend/.sqlx/query-ffde6e45841090440bc99e22788d313db5945faa0b93c127833e44d38665a392.json b/backend/.sqlx/query-ffde6e45841090440bc99e22788d313db5945faa0b93c127833e44d38665a392.json new file mode 100644 index 0000000000..c1eb5fa826 --- /dev/null +++ b/backend/.sqlx/query-ffde6e45841090440bc99e22788d313db5945faa0b93c127833e44d38665a392.json @@ -0,0 +1,30 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT manifest_key, run_results_key FROM dbt_environment_state\n WHERE workspace_id = $1 AND script_path = $2 AND environment = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "manifest_key", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "run_results_key", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + true, + true + ] + }, + "hash": "ffde6e45841090440bc99e22788d313db5945faa0b93c127833e44d38665a392" +} diff --git a/backend/migrations/20260904135713_dbt_environment_state.down.sql b/backend/migrations/20260904135713_dbt_environment_state.down.sql new file mode 100644 index 0000000000..dab9025417 --- /dev/null +++ b/backend/migrations/20260904135713_dbt_environment_state.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS dbt_environment_state; diff --git a/backend/migrations/20260904135713_dbt_environment_state.up.sql b/backend/migrations/20260904135713_dbt_environment_state.up.sql new file mode 100644 index 0000000000..a49df5b3d1 --- /dev/null +++ b/backend/migrations/20260904135713_dbt_environment_state.up.sql @@ -0,0 +1,52 @@ +-- The dbt state one project last built into one environment: the `manifest.json` +-- (and the `run_results.json` beside it) that `dbt --defer --state ` resolves +-- an unbuilt `ref()` through. +-- +-- Separate from `dbt_run_state`, which answers a different question. That one is +-- keyed by the executing principal and holds the LAST run whatever its outcome, +-- so `dbt retry` can resume its failures; this one is keyed by environment and +-- holds the last SUCCESSFUL run, because a relation a later run defers to has to +-- exist. Merging them would make a retry resume a run that is not the last one, +-- or a deferral point at relations a failed run never wrote. +CREATE TABLE IF NOT EXISTS dbt_environment_state ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE ON UPDATE CASCADE, + script_path VARCHAR(255) NOT NULL, + -- The workspace warehouse, the dbt target, and the database and schema that + -- target resolves to. All four, because deferring is resolving a relation + -- NAME: a repointed warehouse or a moved schema makes the stored manifest + -- describe relations that are not where this run would look for them, and the + -- run has no other way to notice. A move therefore reads as an environment + -- with no state yet rather than as state that silently no longer fits. + -- + -- TEXT rather than VARCHAR(255): a project bringing its own `profiles.yml` + -- spells its own schema and database, so the length is the project's. + environment TEXT NOT NULL, + -- The run that published it, so a deferring run can say what it deferred to. + job_id UUID NOT NULL, + -- Exactly one home each. A manifest grows with the project and passes a few + -- hundred KB on a handful of models, so a large one goes to the INSTANCE's + -- object storage and this row keeps the key; a small one stays here, where it + -- costs no round trip and works on an instance that has configured no storage + -- at all. The instance's and not the workspace's, because a member can write + -- the workspace bucket under a key of their choosing, and a manifest is what a + -- later run resolves every unbuilt `ref()` through. `run_results.json` is a + -- tenth of the size and takes the same two homes rather than a rule of its own. + manifest TEXT, + manifest_key TEXT, + run_results TEXT, + run_results_key TEXT, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (workspace_id, script_path, environment), + CONSTRAINT dbt_environment_state_manifest_one_home + CHECK (num_nonnulls(manifest, manifest_key) = 1), + CONSTRAINT dbt_environment_state_run_results_one_home + CHECK (num_nonnulls(run_results, run_results_key) <= 1) +); + +-- No age sweep, unlike the per-run graph rows next door: this table holds one +-- row per script per environment and replaces it in place, so it does not grow +-- with runs, and its reader is every later run of that script — a project that +-- runs monthly must still find last month's state. It goes with the script +-- instead, alongside `dbt_run_state`. +GRANT ALL ON dbt_environment_state TO windmill_user; +GRANT ALL ON dbt_environment_state TO windmill_admin; diff --git a/backend/parsers/windmill-parser-yaml/src/dbt.rs b/backend/parsers/windmill-parser-yaml/src/dbt.rs index 24f1e2c39d..980b547c68 100644 --- a/backend/parsers/windmill-parser-yaml/src/dbt.rs +++ b/backend/parsers/windmill-parser-yaml/src/dbt.rs @@ -58,6 +58,18 @@ impl DbtEngine { pub fn emits_node_events(&self) -> bool { matches!(self, DbtEngine::DbtCore1x) } + + /// Whether the engine has `--defer-state`, the deferral-only half of + /// `--state`. + /// + /// It matters on one command. `dbt retry` reads the run it resumes from + /// `--state`, so an engine with only that flag cannot be told to defer and + /// to resume from the job's own results at once: handed the deferral's + /// directory, it resumes the all-green run stored there and rebuilds + /// nothing. Only dbt-core 1.x separates the two. + pub fn has_defer_state_flag(&self) -> bool { + matches!(self, DbtEngine::DbtCore1x) + } } /// How the warehouse connection is supplied. Both paths are supported @@ -137,6 +149,16 @@ pub struct DbtDescriptor { pub threads: Option, #[serde(default)] pub full_refresh: bool, + /// Resolve a `ref()` a run does not build through the state the last + /// successful run of this environment published, rather than through the + /// schema that run writes into. + /// + /// Only the default for the `build` block's own `defer`, since the choice is + /// per run: the run that publishes an environment's state and the run that + /// defers to it are two invocations of ONE script (decision 6), so a project + /// that could only defer by descriptor could never populate what it reads. + #[serde(default)] + pub defer: bool, /// Automatic in-job retry of the nodes a build failed on. /// /// dbt already confines a failure to its own subtree, and `dbt retry` @@ -258,6 +280,7 @@ pub const RESERVED_ARG_NAMES: &[&str] = &[ "exclude", "vars", "full_refresh", + "defer", "dbt_command", "dbt_retry_job", "model", @@ -345,15 +368,26 @@ fn command_variants(d: &DbtDescriptor) -> Vec<(&'static str, Vec)> { "build", selection() .into_iter() - .chain([Arg { - name: "full_refresh".to_string(), - otyp: None, - typ: Typ::Bool, - has_default: true, - default: Some(serde_json::json!(d.full_refresh)), - oidx: None, - otyp_inferred: false, - }]) + .chain([ + Arg { + name: "full_refresh".to_string(), + otyp: None, + typ: Typ::Bool, + has_default: true, + default: Some(serde_json::json!(d.full_refresh)), + oidx: None, + otyp_inferred: false, + }, + Arg { + name: "defer".to_string(), + otyp: None, + typ: Typ::Bool, + has_default: true, + default: Some(serde_json::json!(d.defer)), + oidx: None, + otyp_inferred: false, + }, + ]) .collect(), ), ( @@ -546,7 +580,9 @@ fn property_of(arg: &Arg) -> serde_json::Value { ), "select" => Some( "dbt selection syntax, e.g. `tag:nightly`, `stg_orders+`, \ - `config.materialized:incremental`. Empty runs the descriptor's own selection.", + `config.materialized:incremental`. `state:modified+` and `result:error+` \ + compare against the state a previous run published, so they need `defer` on. \ + Empty runs the descriptor's own selection.", ), "exclude" => Some("Nodes to leave out of the selection above, same syntax."), "vars" => Some( @@ -554,6 +590,11 @@ fn property_of(arg: &Arg) -> serde_json::Value { exist makes this run store its own graph rather than the deployed one.", ), "full_refresh" => Some("Rebuild incremental models from scratch instead of appending."), + "defer" => Some( + "Resolve a `ref()` this run does not build to the relation the last successful \ + run of this warehouse and target published, instead of to the schema this run \ + writes into.", + ), "model" => Some( "The model to preview, by name — `stg_orders`, or `my_package.stg_orders` when \ two packages share a name. Any dbt selector resolving to ONE node works.", @@ -687,8 +728,15 @@ full_refresh: true }; let (build, build_args) = of("build"); - assert_eq!(build_args, ["exclude", "full_refresh", "select", "vars"]); + assert_eq!( + build_args, + ["defer", "exclude", "full_refresh", "select", "vars"] + ); assert_eq!(build["properties"]["full_refresh"]["type"], "boolean"); + // `defer` is a per-run toggle rather than a descriptor-only setting: the + // run that publishes an environment's state and the run that defers to + // it are two invocations of ONE script. + assert_eq!(build["properties"]["defer"]["type"], "boolean"); // Defaults come from the descriptor, so an untouched run reproduces it. assert_eq!( build["properties"]["select"]["default"], diff --git a/backend/summarized_schema.txt b/backend/summarized_schema.txt index 61f8a66f85..5ab10d1e02 100644 --- a/backend/summarized_schema.txt +++ b/backend/summarized_schema.txt @@ -74,6 +74,8 @@ dbt_edge: workspace_id(char), script_path(char), script_hash(bigint), job_id(uui FK: (workspace_id) -> workspace(id), (workspace_id, script_hash) -> script(workspace_id, hash) dbt_graph_snapshot: workspace_id(char), script_path(char), script_hash(bigint), job_id(uuid), digest(text), relation_root_at_last_ingest(text), ingested_at(ts), permissioned_as(char) FK: (workspace_id) -> workspace(id), (workspace_id, script_hash) -> script(workspace_id, hash) +dbt_environment_state: workspace_id(char), script_path(char), environment(text), job_id(uuid), manifest(text), manifest_key(text), run_results(text), run_results_key(text), updated_at(ts) + FK: (workspace_id) -> workspace(id) dbt_node: workspace_id(char), script_path(char), script_hash(bigint), job_id(uuid), unique_id(text), resource_type(text), name(text), asset_path(text), materialized(text), materialize_strategy(text), unique_key(text), tags(text[]), description(text), test_kind(text), test_column(text), test_args(jsonb), severity(text), attached_node(text), columns(jsonb), freshness(jsonb), raw_code(text), original_file_path(text), ingested_at(ts) FK: (workspace_id) -> workspace(id), (workspace_id, script_hash) -> script(workspace_id, hash) dbt_run_progress: workspace_id(char), job_id(uuid), asset_kind(asset_kind), asset_path(char), status(materialization_status), row_count(bigint), error(text), updated_at(ts) diff --git a/backend/windmill-api-scripts/src/scripts.rs b/backend/windmill-api-scripts/src/scripts.rs index d24b3c3a5c..97e3019835 100644 --- a/backend/windmill-api-scripts/src/scripts.rs +++ b/backend/windmill-api-scripts/src/scripts.rs @@ -2454,27 +2454,28 @@ async fn create_script_internal<'c>( // while its own finished runs still render from them. Clearing by path // would empty those run pages for good. if ns.language != ScriptLang::Dbt { - // The saved retry state does go: nothing regenerates it, it is keyed by - // path alone, and it carries one user's failed invocation and its - // arguments. No dbt version is live at this path any more to resume it. - windmill_common::dbt_manifest::clear_dbt_run_state(&mut tx, &w_id, &ns.path).await?; + // The saved run and environment state do go: nothing regenerates them, + // both are keyed by path alone, and they carry one user's failed + // invocation with its arguments and the project's own manifest. No dbt + // version is live at this path any more to resume or defer to. + windmill_common::dbt_manifest::clear_dbt_script_state(&mut tx, &w_id, &ns.path).await?; } if let Some(ref old) = p_path_opt { if old != &ns.path { clear_script_triggers(&mut *tx, &w_id, old, AssetUsageKind::Script).await?; clear_static_asset_usage(&mut *tx, &w_id, old, AssetUsageKind::Script).await?; - // The saved retry state travels rather than being cleared: nothing + // The saved state travels rather than being cleared: nothing // regenerates it, so dropping it would throw away a resumable - // failure for what is only a rename. Only while the destination is - // still dbt — a rename that also converts the language would - // otherwise reinstate at the new path the state the branch above - // just cleared, leaving one user's arguments and results under a - // path no dbt script occupies. + // failure and every deferral until the next full run, for what is + // only a rename. Only while the destination is still dbt — a rename + // that also converts the language would otherwise reinstate at the + // new path the state the branch above just cleared, leaving one + // user's arguments and results under a path no dbt script occupies. if ns.language == ScriptLang::Dbt { - windmill_common::dbt_manifest::move_dbt_run_state(&mut tx, &w_id, old, &ns.path) + windmill_common::dbt_manifest::move_dbt_script_state(&mut tx, &w_id, old, &ns.path) .await?; } else { - windmill_common::dbt_manifest::clear_dbt_run_state(&mut tx, &w_id, old).await?; + windmill_common::dbt_manifest::clear_dbt_script_state(&mut tx, &w_id, old).await?; } } } @@ -3712,7 +3713,11 @@ async fn archive_script_by_path( path, &w_id ) - .fetch_one(&db) + // In the SAME transaction as the cleanup below, as the by-hash routes are: + // committed on its own, a cleanup that then fails leaves dbt state at a path + // no live version occupies, for whatever is created there next to defer + // through. + .fetch_one(&mut *tx) .await .map_err(|e| Error::internal_err(format!("archiving script in {w_id}: {e:#}")))?; @@ -3720,9 +3725,10 @@ async fn archive_script_by_path( // The graph stays: the pinned read resolves versions through a CTE that // already skips archived rows, so it stops answering for current relations // either way, while deleting it would empty the Models panel of every - // completed run of the project. Retry state does go — nothing may resume a - // script that is no longer live. - windmill_common::dbt_manifest::clear_dbt_run_state(&mut tx, &w_id, path).await?; + // completed run of the project. The saved run and environment state do go — + // nothing may resume a script that is no longer live, and nothing may defer + // through what it last built. + windmill_common::dbt_manifest::clear_dbt_script_state(&mut tx, &w_id, path).await?; // Pipeline event hygiene: an archived script must not be triggered by // anything. Wipe declared `// on ...` edges (asset-event subscribers // look these up). @@ -3807,7 +3813,7 @@ async fn archive_script_by_hash( clear_static_asset_usage_by_script_hash(&mut *tx, &w_id, hash).await?; // The version's graph stays: its finished runs still render from it, and // the live-version CTE already skips archived rows. Deletion clears it. - windmill_common::dbt_manifest::clear_dbt_run_state_if_path_retired( + windmill_common::dbt_manifest::clear_dbt_script_state_if_path_retired( &mut tx, &w_id, &script.path, @@ -3870,7 +3876,12 @@ async fn delete_script_by_hash( ) .bind(&hash.0) .bind(&w_id) - .fetch_one(&db) + // In the SAME transaction as the cleanup below, as `archive_script_by_hash` + // already does. Committed on its own, it opens a window where the path has + // no live version and a concurrent deploy can take it — and the retirement + // guard below then finds that new script live, keeps the old project's dbt + // state, and leaves the replacement able to defer through its manifest. + .fetch_one(&mut *tx) .await .map_err(|e| Error::internal_err(format!("deleting script by hash {w_id}: {e:#}")))?; @@ -3883,7 +3894,7 @@ async fn delete_script_by_hash( windmill_common::dbt_manifest::clear_dbt_manifest_version(&mut tx, &w_id, &script.path, hash.0) .await?; clear_static_asset_usage_by_script_hash(&mut *tx, &w_id, hash).await?; - windmill_common::dbt_manifest::clear_dbt_run_state_if_path_retired( + windmill_common::dbt_manifest::clear_dbt_script_state_if_path_retired( &mut tx, &w_id, &script.path, @@ -3984,11 +3995,11 @@ async fn delete_script_by_path( // After the DELETE, never before: every dbt writer locks the `script` row // first, so taking a sidecar ahead of it deadlocks one of the pair. The - // VERSIONED graph needs no clear at all, cascading off `script`; the retry - // state does, being keyed by path alone and so inherited by whatever is - // created here next, and so do the editor's own graphs, whose NULL + // VERSIONED graph needs no clear at all, cascading off `script`; the saved + // run and environment state do, being keyed by path alone and so inherited + // by whatever is created here next, and so do the editor's own graphs, whose NULL // `script_hash` satisfies that foreign key without riding its cascade. - windmill_common::dbt_manifest::clear_dbt_run_state(&mut tx, &w_id, path).await?; + windmill_common::dbt_manifest::clear_dbt_script_state(&mut tx, &w_id, path).await?; windmill_common::dbt_manifest::clear_dbt_editor_graphs(&mut tx, &w_id, path).await?; if !trash_scripts.is_empty() { @@ -4157,7 +4168,7 @@ async fn delete_scripts_bulk( // Same reason as the single-path delete, over every requested path rather // than the deleted ones: a path that had no script left can still hold state. for p in &request.paths { - windmill_common::dbt_manifest::clear_dbt_run_state(&mut tx, &w_id, p).await?; + windmill_common::dbt_manifest::clear_dbt_script_state(&mut tx, &w_id, p).await?; windmill_common::dbt_manifest::clear_dbt_editor_graphs(&mut tx, &w_id, p).await?; } diff --git a/backend/windmill-common/src/dbt_manifest.rs b/backend/windmill-common/src/dbt_manifest.rs index e85dfc1910..22ac6ddf33 100644 --- a/backend/windmill-common/src/dbt_manifest.rs +++ b/backend/windmill-common/src/dbt_manifest.rs @@ -35,8 +35,8 @@ //! Every `pub` mutator in this module — the manifest ones //! (`replace_dbt_manifest`, `clear_dbt_manifest_version`, //! `clear_dbt_editor_graphs`), -//! the snapshot sweep, and the retry-state ones (`move_dbt_run_state`, -//! `clear_dbt_run_state`, `clear_dbt_run_state_if_path_retired`) — takes the +//! the snapshot sweep, and the script-state ones (`move_dbt_script_state`, +//! `clear_dbt_script_state`, `clear_dbt_script_state_if_path_retired`) — takes the //! workspace and the script to act on as plain arguments and enforces nothing: //! **the caller must already have verified write access to that script**, //! exactly like the sibling `assets::replace_static_asset_usage` each is called @@ -1120,22 +1120,27 @@ pub async fn clear_dbt_editor_graphs( Ok(()) } -/// Move a dbt script's saved retry state to its new path. +/// Move a dbt script's saved state to its new path: the run `dbt retry` resumes, +/// and the state each environment's deferrals resolve through. /// -/// Keyed by path like the sidecar, but unlike the sidecar it is not -/// regenerated by anything: the deploy re-ingests a manifest, while these are -/// the results of a run that already happened. Clearing on rename would throw -/// away a resumable failure for a cosmetic change, so it travels instead. +/// Keyed by path like the sidecar, but unlike the sidecar neither is regenerated +/// by anything: the deploy re-ingests a manifest, while these are the results of +/// runs that already happened. Clearing on rename would throw away a resumable +/// failure, and every deferral until the next full run, for a cosmetic change — +/// so they travel instead. An artifact too large for its row is unaffected: its +/// key is that publication's own, and the moved row is what names it. /// /// See the mutator contract above: this authorizes nothing. -pub async fn move_dbt_run_state( +pub async fn move_dbt_script_state( tx: &mut Transaction<'_, Postgres>, workspace_id: &str, old_path: &str, new_path: &str, ) -> Result<()> { // The destination may already hold state from a script that lived there - // before; the incoming row is the newer truth for this project. + // before; the incoming row is the newer truth for this project. What the + // displaced row named in object storage is left there, as a cleared one's is + // — see `clear_dbt_script_state`. sqlx::query!( "DELETE FROM dbt_run_state WHERE workspace_id = $1 AND script_path = $2", workspace_id, @@ -1151,22 +1156,38 @@ pub async fn move_dbt_run_state( ) .execute(&mut **tx) .await?; + sqlx::query!( + "DELETE FROM dbt_environment_state WHERE workspace_id = $1 AND script_path = $2", + workspace_id, + new_path + ) + .execute(&mut **tx) + .await?; + sqlx::query!( + "UPDATE dbt_environment_state SET script_path = $3 + WHERE workspace_id = $1 AND script_path = $2", + workspace_id, + old_path, + new_path + ) + .execute(&mut **tx) + .await?; Ok(()) } -/// Drop the saved retry state, but only once NO live version of the path is -/// left. +/// Drop the saved state, but only once NO live version of the path is left. /// -/// `dbt_run_state`'s key is the path and the principal — one saved run per script -/// per identity it executes as, not -/// one per version — so archiving or deleting a single version must not take it -/// with them: the live version's `dbt retry` would be refused and the -/// partial-failure resume lost. It does not need to be version-scoped either, -/// because `identity` already refuses a resume whose project, warehouse or -/// engine moved. +/// Neither table is keyed by version — `dbt_run_state` by path and principal, +/// `dbt_environment_state` by path and environment — so archiving or deleting a +/// single version must not take them with it: the live version's `dbt retry` +/// would be refused, its partial-failure resume lost, and every deferral would +/// have to wait for another full run to republish. Neither needs to be +/// version-scoped either: `identity` already refuses a resume whose project, +/// warehouse or engine moved, and a deferral resolves relation names, which a +/// new version of the same project spells the same way. /// /// See the mutator contract above: this authorizes nothing. -pub async fn clear_dbt_run_state_if_path_retired( +pub async fn clear_dbt_script_state_if_path_retired( tx: &mut Transaction<'_, Postgres>, workspace_id: &str, script_path: &str, @@ -1181,17 +1202,34 @@ pub async fn clear_dbt_run_state_if_path_retired( ) .execute(&mut **tx) .await?; + sqlx::query!( + "DELETE FROM dbt_environment_state WHERE workspace_id = $1 AND script_path = $2 + AND NOT EXISTS (SELECT 1 FROM script + WHERE workspace_id = $1 AND path = $2 + AND deleted = false AND archived = false)", + workspace_id, + script_path + ) + .execute(&mut **tx) + .await?; Ok(()) } -/// Drop a dbt script's saved retry state. +/// Drop a dbt script's saved state, both halves. /// -/// Archive and delete: `run_results` is not small, the invocation arguments it -/// carries are the user's, and a script later created at the same path would -/// otherwise inherit a stranger's resumable failure. +/// Archive and delete: neither is small, the invocation arguments and manifest +/// they carry are the user's, and a script later created at the same path would +/// otherwise inherit a stranger's resumable failure and defer to a project it +/// has nothing to do with. +/// +/// An artifact too large for its row lives in the instance's object storage, and +/// this leaves it there — as a deleted script leaves its bundle. Reaching it from +/// here would mean an object-store client in this crate and a delete that has to +/// land after the caller's transaction commits, for one object per environment of +/// a script that is gone. /// /// See the mutator contract above: this authorizes nothing. -pub async fn clear_dbt_run_state( +pub async fn clear_dbt_script_state( tx: &mut Transaction<'_, Postgres>, workspace_id: &str, script_path: &str, @@ -1203,6 +1241,13 @@ pub async fn clear_dbt_run_state( ) .execute(&mut **tx) .await?; + sqlx::query!( + "DELETE FROM dbt_environment_state WHERE workspace_id = $1 AND script_path = $2", + workspace_id, + script_path + ) + .execute(&mut **tx) + .await?; Ok(()) } diff --git a/backend/windmill-common/tests/dbt_graph_storage.rs b/backend/windmill-common/tests/dbt_graph_storage.rs index 7b4971312a..b6c1a1287a 100644 --- a/backend/windmill-common/tests/dbt_graph_storage.rs +++ b/backend/windmill-common/tests/dbt_graph_storage.rs @@ -7,7 +7,8 @@ use sqlx::{Pool, Postgres}; use windmill_common::dbt_manifest::{ - clear_dbt_editor_graphs, clear_dbt_manifest_version, prune_dbt_run_graphs, + clear_dbt_editor_graphs, clear_dbt_manifest_version, clear_dbt_script_state, + clear_dbt_script_state_if_path_retired, move_dbt_script_state, prune_dbt_run_graphs, replace_dbt_editor_graph, replace_dbt_manifest, IngestedManifest, IngestedNode, DBT_EDITOR_GRAPHS_KEPT, DEPLOYED_GRAPH, DEPLOYED_GRAPH_VERSIONS_KEPT, }; @@ -235,7 +236,11 @@ async fn clearing_one_version_leaves_the_others(db: Pool) { // this is where two versions coexist: it pins the batched edge insert // against a real database as well as the version scoping. assert_eq!(edges_for(&db, 1).await, 0, "the cleared version's edges go"); - assert_eq!(edges_for(&db, 2).await, 1, "the other version keeps its own"); + assert_eq!( + edges_for(&db, 2).await, + 1, + "the other version keeps its own" + ); } /// The routes that hard-delete a path clear no graph rows: they delete the @@ -363,7 +368,11 @@ async fn only_the_newest_deploys_keep_their_graph(db: Pool) { // The newest is always among them: losing the live version's graph would // empty the page of every run of it. assert_eq!(nodes_for(&db, over, DEPLOYED_GRAPH).await, 1); - assert_eq!(nodes_for(&db, 1, DEPLOYED_GRAPH).await, 0, "the oldest is reclaimed"); + assert_eq!( + nodes_for(&db, 1, DEPLOYED_GRAPH).await, + 0, + "the oldest is reclaimed" + ); } /// The third provenance: a `parse` of the EDITOR's buffer, which names no @@ -480,17 +489,27 @@ async fn a_version_clear_spares_editor_graphs_and_a_path_clear_does_not(db: Pool replace_dbt_editor_graph(&mut tx, WS, PATH, job, ME, &manifest(&["a"]), "root") .await .unwrap(); - clear_dbt_manifest_version(&mut tx, WS, PATH, 1).await.unwrap(); + clear_dbt_manifest_version(&mut tx, WS, PATH, 1) + .await + .unwrap(); tx.commit().await.unwrap(); assert_eq!(nodes_for(&db, 1, DEPLOYED_GRAPH).await, 0); - assert_eq!(editor_nodes(&db, job).await, 1, "the buffer's graph survives"); + assert_eq!( + editor_nodes(&db, job).await, + 1, + "the buffer's graph survives" + ); let mut tx = db.begin().await.unwrap(); clear_dbt_editor_graphs(&mut tx, WS, PATH).await.unwrap(); tx.commit().await.unwrap(); - assert_eq!(editor_nodes(&db, job).await, 0, "retiring the path takes it"); + assert_eq!( + editor_nodes(&db, job).await, + 0, + "retiring the path takes it" + ); } /// A preview names its own PATH and needs only `jobs:run`, so a bound over the @@ -552,3 +571,157 @@ async fn editor_markers(db: &Pool) -> i64 { .unwrap() .unwrap_or(0) } + +/// A deferral resolves a `ref()` through the manifest of the last successful run +/// at this path, so that state has to follow the script the way the retry state +/// does: a rename must not strand it, and a path no live dbt version occupies +/// must not hand its manifest to whatever is created there next. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn environment_state_follows_the_script(db: Pool) { + const MOVED: &str = "f/test/renamed"; + deploy_script(&db, 1).await; + publish_environment_state(&db, PATH).await; + + let mut tx = db.begin().await.unwrap(); + move_dbt_script_state(&mut tx, WS, PATH, MOVED) + .await + .unwrap(); + tx.commit().await.unwrap(); + assert_eq!(environment_states(&db, PATH).await, 0); + assert_eq!(environment_states(&db, MOVED).await, 1); + + let mut tx = db.begin().await.unwrap(); + clear_dbt_script_state(&mut tx, WS, MOVED).await.unwrap(); + tx.commit().await.unwrap(); + assert_eq!(environment_states(&db, MOVED).await, 0); +} + +/// Archiving or deleting ONE version must not take the path's state with it — +/// the live version's next deferral still needs it — while the last one leaving +/// must, or a script later created at that path inherits the previous project's +/// manifest. The condition is a `NOT EXISTS` in raw SQL, so both directions are +/// pinned against a real database. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn state_goes_only_once_no_live_version_is_left(db: Pool) { + deploy_script(&db, 1).await; + deploy_script(&db, 2).await; + publish_environment_state(&db, PATH).await; + + retire(&db, 1).await; + let mut tx = db.begin().await.unwrap(); + clear_dbt_script_state_if_path_retired(&mut tx, WS, PATH) + .await + .unwrap(); + tx.commit().await.unwrap(); + assert_eq!( + environment_states(&db, PATH).await, + 1, + "another version is still live here" + ); + + retire(&db, 2).await; + let mut tx = db.begin().await.unwrap(); + clear_dbt_script_state_if_path_retired(&mut tx, WS, PATH) + .await + .unwrap(); + tx.commit().await.unwrap(); + assert_eq!( + environment_states(&db, PATH).await, + 0, + "the last one leaving takes it" + ); +} + +async fn retire(db: &Pool, hash: i64) { + sqlx::query!( + "UPDATE script SET archived = true WHERE workspace_id = $1 AND hash = $2", + WS, + hash + ) + .execute(db) + .await + .unwrap(); +} + +/// The worker publishes under a guard naming the version that ran, and the whole +/// point of it is a job that finishes late: its script can be renamed away and an +/// unrelated one created at the same path while it runs, and that project must +/// not inherit this one's manifest as its deferral state. Enforced in raw SQL, +/// where a refactor can drop a predicate with no type error, so it is pinned +/// against a real database — the same shape `dbt_state::publish` issues. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn a_late_job_cannot_publish_for_a_path_it_no_longer_owns(db: Pool) { + deploy_script(&db, 1).await; + assert_eq!(guarded_publish(&db, PATH, 1).await, 1, "its own version"); + assert_eq!( + guarded_publish(&db, PATH, 2).await, + 0, + "a version that never lived here" + ); + + // The script is gone from this path and another one takes it. + sqlx::query!( + "DELETE FROM script WHERE workspace_id = $1 AND hash = 1", + WS + ) + .execute(&db) + .await + .unwrap(); + deploy_script(&db, 3).await; + assert_eq!( + guarded_publish(&db, PATH, 1).await, + 0, + "the late job's version does not own this path any more" + ); +} + +/// The predicate `dbt_state::publish` locks the script row on, reduced to what it +/// decides. Keep the two in step — this file cannot call `publish` itself, which +/// is `pub(crate)` in `windmill-worker`. +async fn guarded_publish(db: &Pool, path: &str, ran: i64) -> u64 { + sqlx::query!( + "INSERT INTO dbt_environment_state (workspace_id, script_path, environment, job_id, + manifest) + SELECT $1::varchar, $2::varchar, 'main||analytics|wh'::text, $3::uuid, '{}'::text + WHERE EXISTS (SELECT 1 FROM script + WHERE workspace_id = $1 AND path = $2 + AND deleted = false AND archived = false + AND language = 'dbt' + AND (hash = $4 OR $4 = ANY(parent_hashes))) + ON CONFLICT (workspace_id, script_path, environment) DO UPDATE SET job_id = EXCLUDED.job_id", + WS, + path, + uuid::Uuid::from_u128(9), + ran, + ) + .execute(db) + .await + .unwrap() + .rows_affected() +} + +async fn publish_environment_state(db: &Pool, path: &str) { + sqlx::query!( + "INSERT INTO dbt_environment_state (workspace_id, script_path, environment, job_id, + manifest) + VALUES ($1, $2, 'main||analytics|wh', $3, '{}')", + WS, + path, + uuid::Uuid::from_u128(9), + ) + .execute(db) + .await + .unwrap(); +} + +async fn environment_states(db: &Pool, path: &str) -> i64 { + sqlx::query_scalar!( + "SELECT count(*) FROM dbt_environment_state WHERE workspace_id = $1 AND script_path = $2", + WS, + path + ) + .fetch_one(db) + .await + .unwrap() + .unwrap_or(0) +} diff --git a/backend/windmill-worker/src/dbt_executor.rs b/backend/windmill-worker/src/dbt_executor.rs index f302df6a1a..727ed78c5c 100644 --- a/backend/windmill-worker/src/dbt_executor.rs +++ b/backend/windmill-worker/src/dbt_executor.rs @@ -19,12 +19,13 @@ use tokio::process::Command; use uuid::Uuid; use windmill_common::client::AuthedClient; use windmill_common::error::{self, Error}; +use windmill_common::jobs::JobKind; use windmill_common::materialization::{ record_materialization, MaterializationStatus, RecordMaterializationRequest, }; use windmill_common::worker::{to_raw_value, write_file, Connection}; use windmill_parser_yaml::{ - parse_dbt_descriptor, DbtDescriptor, DbtTestBehavior, DBT_COMMANDS, DBT_COMMAND_ARG, + parse_dbt_descriptor, DbtDescriptor, DbtEngine, DbtTestBehavior, DBT_COMMANDS, DBT_COMMAND_ARG, DBT_COMMAND_LABEL, DBT_DEFAULT_WAREHOUSE, }; use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; @@ -37,6 +38,9 @@ use crate::dbt_engine::{provision_engine, ProvisionedEngine, DBT_CACHE_DIR}; use crate::dbt_profiles::{ ensure_adapter_licensed, render_dbt_profile, render_profile, DbtAdapter, KnownAdapter, }; +use crate::dbt_state::{ + environment_label, prepare_deferral, write_state_dir, Deferral, StateManifest, STATE_DIR, +}; use crate::handle_child::{ get_mem_peak, handle_child, run_future_with_polling_update_job_poller, JobCtx, JobDeadline, }; @@ -121,6 +125,12 @@ pub struct DbtRunResult { /// the same project — cannot get them from the job. #[serde(skip_serializing_if = "std::collections::HashMap::is_empty")] pub invocation_args: std::collections::HashMap>, + /// The run whose stored state this one resolved its unbuilt `ref()`s + /// through, absent when it deferred to none. What a deferring run built + /// against is otherwise unrecoverable: the state is replaced by the next + /// successful run of that environment. + #[serde(skip_serializing_if = "Option::is_none")] + pub deferred_to: Option, } #[derive(Serialize, Debug, Default)] @@ -189,7 +199,13 @@ pub(crate) async fn handle_dbt_job( // result publishes, and both describe an invocation of this script, not one // executor's view of it. let raw_args = job.args.as_ref().map(|a| a.0.clone()).unwrap_or_default(); - let inv = Invocation { args: args.clone(), raw_args, envs: envs.clone(), strict: true }; + let inv = Invocation { + args: args.clone(), + raw_args, + envs: envs.clone(), + deferral: None, + strict: true, + }; // One wall clock for the whole job. A dbt job is a sequence of // subprocesses — provision, deps, parse, ls, build, then the // `after_all` tests — and each would otherwise resolve the job's full @@ -262,6 +278,16 @@ pub(crate) async fn handle_dbt_job( // applies — nothing is built, so there is no test phase, no materialization, // no retry state and no ownership to publish. if command == "parse" { + // Checked here rather than at the seam below, which cannot tell a parse + // from a run that simply left `defer` off: a parse never reaches the + // deferral at all, so it is the one caller for which "turn `defer` on" + // would be advice that leads nowhere. + check_state_selectors( + &effective_select(&descriptor, &inv)?, + &effective_exclude(&descriptor, &inv)?, + StateAccess::Never(&command), + !selection_is_overridden(&descriptor, &inv.args)?, + )?; return run_parse_only( &prepared, &descriptor, @@ -364,6 +390,84 @@ pub(crate) async fn handle_dbt_job( inv }; + // Read AFTER the retry restore, so a retry defers exactly as the run it + // resumes did: a retry's own arguments are the command block alone, and the + // relations its unbuilt `ref()`s resolve to must not depend on that. + let defer = arg_bool(&inv.args, "defer")?.unwrap_or(descriptor.defer); + // Before the state is fetched, not only at the seam where the selection + // reaches dbt: a selector that cannot work whatever the state says would + // otherwise be masked by the "nothing published yet" refusal, which sends the + // caller to publish a state that will not help. + check_state_selectors( + &effective_select(&descriptor, &inv)?, + &effective_exclude(&descriptor, &inv)?, + if defer { + StateAccess::Given + } else { + StateAccess::OnRequest + }, + !selection_is_overridden(&descriptor, &inv.args)?, + )?; + // A `show` defers too, and every engine takes the flags on it: it COMPILES + // the model it previews, so a model whose upstream this environment built and + // this run did not is exactly the case a deferral exists for. + let inv = if defer { + // Refused before anything runs. `dbt retry` reads the run it resumes + // from `--state`, the flag a deferral needs, so an engine without + // `--defer-state` can be given one or the other: told to defer, it + // resumes the stored state's own (successful) results and rebuilds + // nothing, and left alone it rebuilds the failed nodes with every + // `ref()` resolving into the schema THIS run writes — which for the + // narrowed run a deferral exists to serve is not where those models go. + if command == "retry" && !prepared.engine.engine.has_defer_state_flag() { + return Err(Error::BadRequest(format!( + "`{}` cannot resume a run that deferred: `dbt retry` takes the run it resumes \ + from `--state`, which is also where a deferral reads its manifest, and this \ + engine has no `--defer-state` to tell the two apart. Run the script again \ + instead of resuming it, or move the project to dbt-core-1x", + prepared.engine.engine.as_str() + ))); + } + let deferral = prepare_deferral(&prepared, &job.workspace_id, job_dir, conn).await?; + // Only answerable once the state is loaded: `defer` is enough for a + // `state:` method, which reads the manifest every publication carries, + // but a `result:` one reads `run_results.json` — and a build recovered by + // node retry publishes without it, since the results it holds describe + // only the nodes the retry rebuilt. dbt-core then raises an INTERNAL + // error and the Rust engines match nothing and exit 0. + if !deferral.has_run_results + && selection_names( + &effective_select(&descriptor, &inv)?, + &effective_exclude(&descriptor, &inv)?, + &["result"], + ) + { + return Err(Error::BadRequest(format!( + "a `result:` selector reads `run_results.json` out of the published state, and \ + the state for this environment ({}) carries only the manifest run {} \ + published: a build recovered by node retry stores none, its results describing \ + the retried nodes rather than the whole build. Run this script once without \ + `defer` and without overrides to publish a complete state, or drop the selector", + environment_label(&prepared), + deferral.published_by + ))); + } + append_logs( + &job.id, + &job.workspace_id, + format!( + "\nDeferring unbuilt refs to the dbt state published by run {}; this run \ + publishes none of its own\n", + deferral.published_by + ), + conn, + ) + .await; + Invocation { deferral: Some(deferral), ..inv } + } else { + inv + }; + // Ingested BEFORE the build, from a `dbt parse` with this run's vars, so the // models shown are the ones about to be built. Rows are keyed by path, version // AND job so no two runs collide; the path-keyed `asset` usage belongs to one @@ -428,9 +532,28 @@ pub(crate) async fn handle_dbt_job( // previous attempt's `run_results.json` is still in the job directory. Never on // an agent worker, which cannot read `v2_job_queue` — the wait below would be // uninterruptible, so a cancelled job would hold its slot and then start dbt. + // And never where the engine cannot be told to defer on a `retry`: the + // rebuild would resolve this run's unbuilt refs into the schema it writes + // into, so the nodes it "recovered" would read from the wrong relations. + // Said out loud below rather than silently skipped. + let retry_would_lose_the_deferral = + inv.deferral.is_some() && !prepared.engine.engine.has_defer_state_flag(); let node_retry = descriptor .retry_failed_nodes - .filter(|_| matches!(conn, Connection::Sql(_))); + .filter(|_| matches!(conn, Connection::Sql(_))) + .filter(|_| !retry_would_lose_the_deferral); + if descriptor.retry_failed_nodes.is_some() && retry_would_lose_the_deferral { + append_logs( + &job.id, + &job.workspace_id, + format!( + "\nSkipping the automatic node retry: `{}` cannot defer on a `dbt retry`\n", + prepared.engine.engine.as_str() + ), + conn, + ) + .await; + } let mut retries_left = node_retry.map(|p| p.attempts()).unwrap_or(0); if let Some(policy) = node_retry.filter(|_| run.is_err()) { retry_failed_nodes( @@ -519,6 +642,56 @@ pub(crate) async fn handle_dbt_job( { tracing::warn!("dbt: could not save retry state for job {}: {e:#}", job.id); } + // What a later run defers to, published by the runs whose relations are the + // SCRIPT's — the same condition that decides whether a run's graph becomes + // what the script owns, and for the same reason: an invocation that scoped + // its own models has no standing to say where this project's relations live. + // Success is the other half, because a relation a deferral resolves to has + // to exist. A `retry` is excluded: its `run_results.json` names only the + // nodes it redid, so publishing it would leave the environment claiming a + // run of a handful of models. + // + // And never a run that DEFERRED, whatever narrowed it. A deferring run built + // some of the relations its manifest names and resolved the rest out of the + // state it read, so publishing that manifest would record relations nothing + // built — and a model renamed since would be recorded under a name only a + // full build creates, breaking every later deferral until one repairs it. + // `publishes_ownership` cannot see this on its own: it reads the caller's + // overrides, and a descriptor that already narrows `select` needs none. + if run.is_ok() + && command == "build" + && inv.deferral.is_none() + // A run of the DEPLOYED version, by kind. A preview carries a + // caller-supplied `script_hash` into `runnable_id` + // (`run_preview_script`), so the version guard alone would let anyone who + // may run a job publish arbitrary content as a deployed script's state. + && job.kind == JobKind::Script + && prepared.graph_refresh.publishes_ownership() + { + // Losing it costs the next deferral, not the run that just finished — + // but silently, so the one actionable case (an artifact too large for + // the database on an instance with no object storage) says so. + if let Err(e) = crate::dbt_state::publish( + &prepared, + &job.workspace_id, + &job.id, + job.runnable_id.map(|h| h.0), + // An attempt was spent, so `run_results.json` on disk is the one + // `dbt retry` left: the nodes it redid, not the build. + node_retry.is_some_and(|p| retries_left < p.attempts()), + conn, + ) + .await + { + append_logs( + &job.id, + &job.workspace_id, + format!("\nCould not publish this run as the environment's dbt state: {e}\n"), + conn, + ) + .await; + } + } let reconciled = reconcile_materializations(&prepared, &results, job, conn, client).await; terminalize_running_relations(job, &reconciled, conn).await; @@ -895,6 +1068,17 @@ pub struct PreparedProject { /// The descriptor's `profile.target`, passed as `--target` so it applies to /// a project-owned `profiles.yml` as well as a rendered one. pub target: Option, + /// The target dbt actually runs, which is the above only when the descriptor + /// names one: otherwise it is the workspace warehouse's, or the project's own + /// `profiles.yml` default. Half of an environment's identity, since a + /// `target.name` macro decides where a model is built. + pub effective_target: Option, + /// Whether the profile templates where its relations go — a project-owned + /// `profiles.yml`, a `dbt_profile` resource's block, or `profile.schema`, + /// all of which reach dbt as written. Two renderings then share one + /// `relation_root` and an environment cannot be told apart, so such a + /// project neither publishes state nor defers to any. + pub templated_location: bool, /// The profile target's database. Nodes that override it qualify their /// `dbt://` schema segment so two databases cannot collapse onto one node. pub default_database: Option, @@ -932,7 +1116,7 @@ impl PreparedProject { /// Where this run's relations live: the resolved schema and database. Drift /// here since the deploy means the stored graph names relations that no /// longer exist. - fn relation_root(&self) -> String { + pub(crate) fn relation_root(&self) -> String { format!( "{}|{}", self.default_schema.as_deref().unwrap_or(""), @@ -1064,8 +1248,8 @@ pub(crate) async fn prepare_project( .chain(invocation_env.iter().map(|(k, v)| (k.clone(), v.clone()))) .collect(); - let (profiles_dir, warehouse, adapter, default_database, default_schema, profile_digest) = - write_profiles(descriptor, &project_dir, job_dir, client, &template_env).await?; + let profile = write_profiles(descriptor, &project_dir, job_dir, client, &template_env).await?; + let adapter = profile.adapter.clone(); // The lockfile's version, when it pinned one for this same engine — a // descriptor edited to another engine invalidates the pin. let pinned_version = locks @@ -1186,18 +1370,20 @@ pub(crate) async fn prepare_project( h.finish() }, sandbox_config, - profile_digest, + profile_digest: profile.digest, project_dir, - profiles_dir, + profiles_dir: profile.dir, engine, graph_refresh, - warehouse, + warehouse: profile.warehouse, target: descriptor.profile.target.clone(), + effective_target: profile.target, + templated_location: profile.templated_location, descriptor_content: descriptor_content.to_string(), descriptor_env, - default_database, - default_schema, + default_database: profile.database, + default_schema: profile.schema, script_path: script_path.to_string(), env, }; @@ -1512,6 +1698,24 @@ async fn strip_git_remote(dir: &Path) -> std::io::Result<()> { tokio::fs::write(&config, out).await } +/// What resolving the run's connection settled, beyond the file itself. +struct ResolvedProfile { + dir: PathBuf, + /// The workspace warehouse's NAME, when this project belongs to one. + warehouse: Option, + adapter: DbtAdapter, + database: Option, + schema: Option, + /// The target dbt actually runs, which is not always the descriptor's: it + /// falls back to the workspace warehouse's, and to the project's own + /// `profiles.yml` default. Resolved because it is half of an environment's + /// identity and a `target.name` macro can move every relation. + target: Option, + /// Whether a project-owned `profiles.yml` templates where its relations go. + templated_location: bool, + digest: String, +} + /// Write `profiles.yml`, either rendered from a Windmill resource or taken from /// the project itself. Both paths are supported (decision 8): the workspace /// warehouse is the ergonomic one, the project's own file is what makes an @@ -1522,14 +1726,7 @@ async fn write_profiles( job_dir: &str, client: &AuthedClient, template_env: &HashMap, -) -> error::Result<( - PathBuf, - Option, - DbtAdapter, - Option, - Option, - String, -)> { +) -> error::Result { // The workspace's warehouse, always: a descriptor names one by NAME or takes // `main`, and cannot name a resource at all. The NAME is what asset identity // keys on, so every project on one warehouse shares its nodes while the @@ -1610,14 +1807,16 @@ async fn write_profiles( } None => None, }; - return Ok(( + return Ok(ResolvedProfile { dir, - identity, + warehouse: identity, adapter, - target.database, - target.schema, - profile_digest, - )); + database: target.database, + schema: target.schema, + target: Some(target.name), + templated_location: target.templated_location, + digest: profile_digest, + }); } use windmill_common::workspaces::DBT_PROFILE_RESOURCE_TYPE; @@ -1712,14 +1911,22 @@ async fn write_profiles( rendered.root_certificate_pem.as_deref(), &client.token, ); - Ok(( + Ok(ResolvedProfile { dir, - Some(warehouse.to_string()), + warehouse: Some(warehouse.to_string()), adapter, - rendered.database, - rendered.schema, - profile_digest, - )) + // A `dbt_profile` resource is one block of the user's own + // `profiles.yml`, copied through unchanged, and `profile.schema` is + // written as given — so either can be a template dbt renders and this + // runtime does not, exactly as a project-owned file can. + templated_location: [rendered.database.as_deref(), rendered.schema.as_deref()] + .iter() + .any(|v| v.is_some_and(is_jinja)), + database: rendered.database, + schema: rendered.schema, + target: Some(target.to_string()), + digest: profile_digest, + }) } /// Where a workspace warehouse name points: its resource path and, if the @@ -1860,13 +2067,45 @@ async fn adapter_from_profiles_yml( // identically to one on a workspace warehouse, which is what lets the two // meet on the same node when they are on the same relation. let (database_key, schema_key) = adapter.target_identity_keys(); - let read = |k: &str| { + let raw = |k: &str| { out.get(k) .and_then(|v| v.as_str()) - .map(|v| v.to_string()) - .filter(|v| !v.is_empty() && !v.contains("{{")) + .filter(|v| !v.is_empty()) }; - Ok(ProfileTarget { adapter, database: read(database_key), schema: read(schema_key) }) + let read = |k: &str| raw(k).filter(|v| !v.contains("{{")).map(|v| v.to_string()); + Ok(ProfileTarget { + adapter, + database: read(database_key), + schema: read(schema_key), + // A TEMPLATED location is one dbt renders and this runtime does not, so + // two renderings of this file resolve to one `relation_root` and would + // share one environment — `{{ }}` because `read` drops it and it reads + // as absent, `{% %}` because the raw block is kept and reads the same + // for every rendering. Distinguished from plainly absent, which is the + // adapter's default and does not move. + templated_location: [database_key, schema_key] + .iter() + .any(|k| raw(k).is_some_and(is_jinja)), + // The output actually chosen, which for a templated `target:` is the sole + // one rather than the template text no output answers to. + name: match ( + templated_target, + outputs.as_mapping().and_then(|m| m.keys().next()), + ) { + (true, Some(only)) => only.as_str().unwrap_or(target).to_string(), + _ => target.to_string(), + }, + }) +} + +/// Whether dbt would RENDER this value rather than take it literally. +/// +/// Both delimiters, because dbt renders a profile through Jinja: `{{ … }}` +/// substitutes and `{% … %}` branches, and a schema spelled +/// `{% if env_var('ENV') == 'prod' %}analytics{% else %}dev{% endif %}` moves +/// every relation exactly as an `env_var()` does. +fn is_jinja(v: &str) -> bool { + v.contains("{{") || v.contains("{%") } /// What a project-owned `profiles.yml` target says, for the two things Windmill @@ -1877,6 +2116,14 @@ struct ProfileTarget { adapter: DbtAdapter, database: Option, schema: Option, + /// The output this resolved to, by name. + name: String, + /// Whether its database or schema is a template rather than a literal. The + /// fields above cannot say: a `{{ }}` value is dropped and reads as absent, + /// a `{% %}` block is kept and reads the same for every rendering. So this + /// is what separates "the adapter's default, which does not move" from + /// "wherever this run's environment renders it to". + templated_location: bool, } lazy_static::lazy_static! { @@ -2192,6 +2439,29 @@ async fn retry_failed_nodes( } } +/// The flags that point a deferring invocation at its state directory. +/// +/// `--state` is where a deferred `ref()` resolves through — except on a `retry`, +/// which reads the run it RESUMES from that same flag: handed the deferral's +/// directory, dbt resumes the successful run stored there and rebuilds nothing. +/// dbt-core 1.x has `--defer-state` for exactly this split; the Rust engines do +/// not, and a run that defers is refused a retry there rather than rebuilt with +/// its refs resolving into the schema it writes into (`handle_dbt_job`), which +/// is why the last arm never fires in practice. +/// +/// The directory is relative because dbt records the invocation's flags into +/// `run_results.json`: an absolute path would name the job directory of the run +/// being resumed, gone by the time anything reads it back. +fn defer_flags(command: &str, engine: DbtEngine) -> &'static [&'static str] { + match command { + // `--defer` itself is restored with the rest of the resumed + // invocation's arguments and cannot be set from here. + "retry" if engine.has_defer_state_flag() => &["--defer-state", STATE_DIR], + "retry" => &[], + _ => &["--defer", "--state", STATE_DIR], + } +} + #[allow(clippy::too_many_arguments)] async fn run_dbt( p: &PreparedProject, @@ -2213,6 +2483,10 @@ async fn run_dbt( .args(["--log-format-file", "json"]) .args(["--log-level-file", p.engine.engine.progress_log_level()]); + if inv.deferral.is_some() { + cmd.args(defer_flags(command, p.engine.engine)); + } + if with_selection && command != "retry" { add_selection(&mut cmd, descriptor, inv)?; } @@ -2864,6 +3138,9 @@ async fn run_show( ))); } let mut cmd = dbt_command(p, &["show"]); + if inv.deferral.is_some() { + cmd.args(defer_flags("show", p.engine.engine)); + } add_vars(&mut cmd, descriptor, inv)?; // Intersected with `resource_type:model`, because `show` is only read-only // for models: dbt dispatches a selected SEED through its seed runner and @@ -2957,6 +3234,7 @@ fn build_result( totals, nodes, invocation_args: inv.raw_args.clone(), + deferred_to: inv.deferral.as_ref().map(|d| d.published_by), } } @@ -3470,6 +3748,12 @@ async fn resolve_selection( return Ok(None); } let mut cmd = dbt_command(p, &["ls"]); + // The same state the build resolves through, or a `result:` selector — which + // reads `run_results.json` out of it, and which `select` passes to dbt + // verbatim — fails here, before the build that would have honoured it. + if inv.deferral.is_some() { + cmd.args(defer_flags("ls", p.engine.engine)); + } // A project whose models call `var()` without a default fails to parse // without these, so the selection resolver needs them exactly as the run // does. Placeholders that only a run can fill are dropped rather than @@ -3483,6 +3767,8 @@ async fn resolve_selection( } cmd.args(["--output", "json", "--quiet"]); add_selection(&mut cmd, descriptor, inv)?; + let select = effective_select(descriptor, inv)?; + let exclude = effective_exclude(descriptor, inv)?; // Captured directly, not through `handle_child`: its `pipe_stdout` path goes // through the job-log writer, which `NO_LOGS_AT_ALL` discards — the selection // would resolve to the empty set and the ingest would wipe the script's assets @@ -3500,15 +3786,37 @@ async fn resolve_selection( } } } - if set.is_empty() { - // A selection that matches nothing would be ingested as "this script - // owns no relations", wiping its graph and cascade edges — the same - // outcome a failed capture produces, and indistinguishable from it. - // Refuse rather than silently un-wire the script. + // Empty is a real answer from a `state:` or `result:` method and from nothing + // else: `state:modified+` matches nothing exactly when nothing changed since + // the published state, and a run with no work to do is a successful one. Any + // other selection matching nothing is a selector that names nothing — a + // misspelled model, say — which must not pass as a build that did its job. + // Exempting by ORIGIN rather than by method would let every such typo through. + // + // What makes the exemption safe is that the empty set is never ingested as + // ownership, and that now holds through `check_state_selectors`: a `state:` + // or `result:` method survives it only from a run's OWN selection, which + // makes `add_caller_args` set `per_run_models`, which makes + // `publishes_ownership()` false, so the run stores a snapshot of its own. + // Relax the descriptor arm there and a descriptor-narrowed `state:modified+` + // reaches here on an unchanged project and wipes the graph the `else` below + // guards, with nothing failing. + if set.is_empty() && !selection_names(&select, &exclude, &["state", "result"]) { return Err(Error::ExecutionErr( - "the descriptor's `select`/`exclude` matched no dbt nodes; fix the selection rather \ - than deploying a script that owns nothing" - .to_string(), + if selection_is_overridden(descriptor, &inv.args)? { + "this run's `select`/`exclude` matched no dbt nodes, so it would build nothing; \ + check the selector. Only a `state:` or `result:` selector may match nothing, \ + its empty answer being a real one" + .to_string() + } else { + // The descriptor's is also ingested as "this script owns no + // relations", wiping its graph and cascade edges — the same + // outcome a failed capture produces, and indistinguishable from + // it. Refuse rather than silently un-wire the script. + "the descriptor's `select`/`exclude` matched no dbt nodes; fix the selection \ + rather than deploying a script that owns nothing" + .to_string() + }, )); } Ok(Some(set)) @@ -3820,7 +4128,7 @@ async fn save_run_state( if let Connection::Sql(db) = conn { { // Only while a live dbt version stays at this path — the test - // `clear_dbt_run_state_if_path_retired` retires state by, plus the + // `clear_dbt_script_state_if_path_retired` retires state by, plus the // language, since a rename leaves the old path archived rather than // deleted and a path can come back as another language. A job already // running finishes after those move or clear the row: writing then @@ -3982,6 +4290,12 @@ pub struct Invocation { /// what it pointed at must not. pub raw_args: HashMap>, pub envs: HashMap, + /// The stored dbt state this invocation resolves an unbuilt `ref()` through, + /// materialised into the job directory. Carried here rather than passed to + /// each phase: the model phase, the `after_all` tests and every in-job node + /// retry must all resolve a `ref()` the same way, or the tests assert against + /// relations the models never read. + pub deferral: Option, /// A run must fail on a `{{ }}` placeholder it cannot fill; a deploy, which /// has no arguments at all, tolerates them. Declared rather than inferred /// from the argument count: a run submitted with `{}` is still a run, and @@ -4134,11 +4448,12 @@ async fn restore_from_db( if !has_retryable_node(&row.run_results) { return Err(nothing_to_retry()); } - let target = p.project_dir.join(ARTIFACTS_DIR); - tokio::fs::create_dir_all(&target).await.ok(); - tokio::fs::write(target.join("run_results.json"), &row.run_results) - .await - .map_err(|e| Error::internal_err(format!("restoring run_results.json: {e}")))?; + write_state_dir( + &p.project_dir.join(ARTIFACTS_DIR), + Some(&row.run_results), + StateManifest::None, + ) + .await?; // No manifest came with the row, so one has to be re-derived — but not here: // these arguments are as SUBMITTED, and a `$var:` in them shapes the graph // only once resolved. The caller resolves, then parses. @@ -4370,20 +4685,16 @@ async fn restore_run_state( return Err(different_project()); } let saved_args_digest = saved_args_digest.map(str::to_string); - let target = p.project_dir.join(ARTIFACTS_DIR); - tokio::fs::create_dir_all(&target).await.ok(); - // From the bytes already read, not by copying the file again: a burst of saves - // can prune this generation mid-restore, and a `dbt retry` whose + // The results go from the bytes already read, not by copying the file again: a + // burst of saves can prune this generation mid-restore, and a `dbt retry` whose // `run_results.json` went missing rebuilds nothing and reports success. The // manifest has no such copy, so a failure there falls back to a `dbt parse`. - tokio::fs::write(target.join("run_results.json"), &saved_results) - .await - .map_err(|e| { - Error::internal_err(format!("could not restore the previous run's results: {e}")) - })?; - let needs_parse = tokio::fs::copy(snapshot.join("manifest.json"), target.join("manifest.json")) - .await - .is_err(); + let needs_parse = !write_state_dir( + &p.project_dir.join(ARTIFACTS_DIR), + Some(&saved_results), + StateManifest::CopyOf(snapshot.join("manifest.json")), + ) + .await?; // The generation was chosen from a row read before the file work above. A run // finishing in that window publishes a newer one, and resuming the superseded // generation redoes nodes it has already rebuilt — appending to an incremental @@ -4772,10 +5083,26 @@ fn add_selection( descriptor: &DbtDescriptor, inv: &Invocation, ) -> error::Result<()> { - for s in effective_select(descriptor, inv)? { + let select = effective_select(descriptor, inv)?; + let exclude = effective_exclude(descriptor, inv)?; + // The seam itself, which the DEPLOY reaches without going through a run: it + // resolves the descriptor's selection to decide what the script owns, and + // never computes a `defer`. A run has been checked earlier, where the message + // can still come before the state fetch. + check_state_selectors( + &select, + &exclude, + if inv.deferral.is_some() { + StateAccess::Given + } else { + StateAccess::OnRequest + }, + !selection_is_overridden(descriptor, &inv.args)?, + )?; + for s in select { cmd.args(["--select", &s]); } - for s in effective_exclude(descriptor, inv)? { + for s in exclude { cmd.args(["--exclude", &s]); } if let Some(sel) = effective_selector(descriptor, inv)? { @@ -4784,6 +5111,115 @@ fn add_selection( Ok(()) } +/// The method a selection token names, with the graph operators that can +/// surround a node stripped (`@model`, `+model`, `2+model`, `model+`). +fn selector_method(token: &str) -> Option<&str> { + token + .trim_start_matches('@') + .trim_start_matches(|c: char| c.is_ascii_digit()) + .trim_start_matches('+') + .split_once(':') + .map(|(method, _)| method) +} + +/// Every method a selection names. Each entry is a union of whitespace-separated +/// tokens, and each of those an intersection of comma-separated ones. +fn selection_methods<'a>(entries: &'a [String]) -> impl Iterator { + entries + .iter() + .flat_map(|entry| entry.split([' ', '\t', ','])) + .filter_map(selector_method) +} + +/// Whether the run being checked has the state directory a `state:` or `result:` +/// method reads, or could be given one. +#[derive(Clone, Copy)] +enum StateAccess<'a> { + /// Deferring, so the directory is there. + Given, + /// Not deferring, and `defer` is what would hand it one. + OnRequest, + /// This command resolves a selection without ever deferring, so no setting + /// gives it a state and "turn `defer` on" would be advice that leads nowhere. + Never(&'a str), +} + +/// Whether a selection names any of these methods. +fn selection_names(select: &[String], exclude: &[String], methods: &[&str]) -> bool { + selection_methods(select) + .chain(selection_methods(exclude)) + .any(|method| methods.contains(&method)) +} + +/// Refuse a selection dbt cannot resolve, before it silently resolves to the +/// wrong thing. +/// +/// `state:` and `result:` compare against the artifacts in `--state`, which only +/// a deferring run is given. The engines do not agree on what happens without +/// one: dbt-core 1.x raises, but dbt-sa-cli and fusion read a missing state as an +/// EMPTY one and exit 0, so `state:modified` builds nothing and `state:new` +/// builds the whole project, each as a run that reports success. +/// +/// From the DESCRIPTOR they are refused whether or not the run defers, because +/// that selection also decides which nodes the script owns, and "whatever changed +/// last" is not an ownership answer — the deploy resolves it with no state at all. +/// They describe one run, so they belong in a run's own `select`. +/// +/// `source_status:` compares `sources.json`, which `dbt source freshness` writes +/// and no run publishes here, so it has nothing to compare against under any +/// setting. +/// +/// Only what `select` and `exclude` spell directly: a method reached through a +/// `selectors.yml` definition is named nowhere the worker can read, and dbt's +/// own behaviour is what stands there. +fn check_state_selectors( + select: &[String], + exclude: &[String], + access: StateAccess<'_>, + from_descriptor: bool, +) -> error::Result<()> { + for method in selection_methods(select).chain(selection_methods(exclude)) { + match method { + "source_status" => { + return Err(Error::BadRequest( + "a `source_status:` selector compares the source freshness recorded in \ + `sources.json`, which `dbt source freshness` writes and no run stores \ + here, so there is nothing for it to compare against. Drop the selector" + .to_string(), + )) + } + "state" | "result" if from_descriptor => { + return Err(Error::BadRequest(format!( + "a `{method}:` selector describes what ONE run builds, but the descriptor's \ + selection also decides which nodes this script owns, which a deploy \ + resolves with no state to compare against. Move it to the `select` of a \ + run with `defer` on" + ))) + } + "state" | "result" => match access { + StateAccess::Given => {} + StateAccess::OnRequest => { + return Err(Error::BadRequest(format!( + "a `{method}:` selector compares against the dbt state a previous run \ + of this environment published, and only a run with `defer` on is given \ + that state. Turn `defer` on, or drop the selector" + ))) + } + StateAccess::Never(command) => { + return Err(Error::BadRequest(format!( + "a `{method}:` selector compares against the dbt state a previous run \ + of this environment published, and `{command}` resolves its selection \ + without building and never defers, so no setting hands it that state. \ + Drop the selector" + ))) + } + }, + _ => {} + } + } + Ok(()) +} + /// Whether this invocation chose its own `select`/`exclude`. /// /// DIFFERENT from the descriptor's, not merely present: `parse_dbt_sig` gives @@ -5797,6 +6233,27 @@ mod tests { .unwrap(); assert!(untouched.publishes_ownership()); assert_eq!(untouched.snapshot_job(job), None); + + // `resolve_selection` lets a selection match nothing on exactly this + // predicate, because a run that scoped its own selection stores a + // snapshot instead of publishing ownership. Should the two ever drift + // apart, an empty caller selection would wipe the script's graph and + // cascade edges, which is the outcome that guard exists to prevent. + // One-directional: a `vars` override also withholds ownership without + // touching the selection, which is why this is an implication and not an + // equivalence. + for args in [ + arg("select", r#"["state:modified+"]"#), + arg("exclude", r#"["tag:nightly"]"#), + ] { + assert!(selection_is_overridden(&descriptor, &args).unwrap()); + let mut g = GraphRefresh::default(); + g.add_caller_args(&descriptor, &args).unwrap(); + assert!( + !g.publishes_ownership(), + "an overridden selection must not publish ownership" + ); + } } // `dbt retry` restores the previous run's target/ from this directory, so two @@ -5825,6 +6282,128 @@ mod tests { ); } + // A profile whose location dbt renders cannot be told apart from another + // rendering of itself, so it neither publishes state nor defers. Both + // delimiters count: a conditional block moves a schema exactly as an + // `env_var()` substitution does. + #[test] + fn a_rendered_profile_location_is_recognised_by_either_delimiter() { + assert!(is_jinja("{{ env_var('DBT_SCHEMA') }}")); + assert!(is_jinja( + "{% if env_var('ENV') == 'prod' %}analytics{% else %}dev{% endif %}" + )); + assert!(!is_jinja("analytics")); + assert!(!is_jinja("")); + } + + // dbt-sa-cli and fusion exit 0 on a state selector with no state, so nothing + // downstream would report this: the graph operators have to be stripped for + // the method to be seen at all. + #[test] + fn a_state_selector_is_found_under_any_graph_operator() { + // A run's own selection, which is the only place these belong. + let refused = |sel: &str, access: StateAccess<'_>| { + check_state_selectors(&[sel.to_string()], &[], access, false).is_err() + }; + for sel in [ + "state:modified", + "state:modified+", + "+state:new", + "@state:modified", + "2+state:modified+3", + "tag:nightly,state:modified", + "stg_orders+ result:error+", + ] { + assert!( + refused(sel, StateAccess::OnRequest), + "{sel} should need `defer`" + ); + assert!( + !refused(sel, StateAccess::Given), + "{sel} should pass while deferring" + ); + // A parse resolves a selection without ever deferring, so it is + // refused where a run would have been told to turn `defer` on. + assert!( + refused(sel, StateAccess::Never("parse")), + "{sel} cannot parse" + ); + // The descriptor's selection also decides what the script owns, and + // the deploy resolves it with no state, so deferring cannot save it. + assert!( + check_state_selectors(&[sel.to_string()], &[], StateAccess::Given, true).is_err(), + "{sel} should never be a descriptor selection" + ); + } + // A node whose name merely starts with a method's letters is not one. + for sel in ["stg_orders+", "tag:nightly", "stateful_model+"] { + assert!( + !refused(sel, StateAccess::OnRequest), + "{sel} is not a state selector" + ); + } + // No run publishes `sources.json`, so deferring does not help. + for access in [ + StateAccess::Given, + StateAccess::OnRequest, + StateAccess::Never("parse"), + ] { + assert!(refused("source_status:fresher+", access)); + } + // `exclude` reaches dbt the same way `select` does. + assert!(check_state_selectors( + &[], + &["state:modified".to_string()], + StateAccess::OnRequest, + false + ) + .is_err()); + + // The same recognition decides which empty selections `resolve_selection` + // lets through. Only these two answer "nothing" meaningfully; a selector + // naming nothing must not pass as a build that did its work. + const STATE_BACKED: &[&str] = &["state", "result"]; + for sel in ["state:modified+", "result:error+", "tag:x,state:new"] { + assert!( + selection_names(&[sel.to_string()], &[], STATE_BACKED), + "{sel}" + ); + } + for sel in [ + "mispelled_model", + "tag:nightly", + "stg_orders+", + "source_status:fresher+", + ] { + assert!( + !selection_names(&[sel.to_string()], &[], STATE_BACKED), + "{sel}" + ); + } + } + + // The one flag choice that is silently wrong rather than loudly wrong: a + // `retry` handed `--state` resumes the SUCCESSFUL run stored there and + // rebuilds nothing, reporting a green retry of a failed run. + #[test] + fn a_retry_is_never_handed_the_deferral_as_its_state() { + assert_eq!( + defer_flags("build", DbtEngine::DbtCore1x), + ["--defer", "--state", crate::dbt_state::STATE_DIR] + ); + assert_eq!( + defer_flags("test", DbtEngine::Fusion), + ["--defer", "--state", crate::dbt_state::STATE_DIR] + ); + assert_eq!( + defer_flags("retry", DbtEngine::DbtCore1x), + ["--defer-state", crate::dbt_state::STATE_DIR] + ); + for engine in [DbtEngine::DbtCore2x, DbtEngine::Fusion] { + assert!(defer_flags("retry", engine).is_empty()); + } + } + #[test] fn events_without_a_relation_are_not_materializations() { // A test node has no relation of its own. diff --git a/backend/windmill-worker/src/dbt_state.rs b/backend/windmill-worker/src/dbt_state.rs new file mode 100644 index 0000000000..ecf3d0dbe3 --- /dev/null +++ b/backend/windmill-worker/src/dbt_state.rs @@ -0,0 +1,748 @@ +//! The dbt state a project last built into one environment, and the state +//! directory a run reads it back through. +//! +//! `dbt --defer --state ` resolves a `ref()` the run does not build to the +//! relation the manifest in `` names, instead of to the schema this run +//! writes into. That makes the state a durable, per-environment artifact rather +//! than a cache: the next run of a project usually lands on a worker holding +//! neither the manifest nor the results, so anything worker-local answers for +//! one machine's history rather than for the environment. +//! +//! Two artifacts live in that directory and both are stored: `manifest.json`, +//! which is what a deferral resolves through, and `run_results.json`, which +//! `select`'s `result:` selectors read — and `select` reaches dbt verbatim, so a +//! state directory missing it fails a selection a user may legitimately write. + +use std::path::{Path, PathBuf}; + +use uuid::Uuid; +use windmill_common::error::{self, Error}; +use windmill_common::worker::Connection; + +use crate::dbt_executor::{digest, PreparedProject, ARTIFACTS_DIR}; + +lazy_static::lazy_static! { + /// Above this, an artifact goes to the instance's object storage instead of + /// into the row. A manifest passes a few hundred KB on a handful of models + /// and grows with the project, so this ceiling is what decides whether a + /// large project needs storage configured at all; a small one stays in the + /// database, where it costs no round trip and needs nothing configured. + static ref DBT_STATE_INLINE_MAX_BYTES: usize = std::env::var("DBT_STATE_INLINE_MAX_BYTES") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(8 * 1024 * 1024); +} + +/// The directory `--state` points at. Inside the job directory, so it sits in +/// the sandbox's one writable bind and goes away with the job, and prefixed like +/// the artifacts directory beside it so a project carrying a directory of this +/// name is not overwritten. +/// +/// Passed to dbt RELATIVE, and that is load-bearing rather than tidiness. dbt +/// records the invocation's flags into `run_results.json` and a later +/// `dbt retry` restores them, so an absolute path would name the job directory +/// of the run being resumed — gone by then, leaving the retry to resolve a +/// deferred `ref()` against nothing. Relative, it resolves against the project +/// root, which is whichever job directory the retry landed in. +pub(crate) const STATE_DIR: &str = "wm_dbt_state"; + +/// Where this run's relations live, which is the only thing a deferral is about. +pub(crate) fn environment(p: &PreparedProject) -> String { + environment_key( + p.warehouse.as_deref(), + // The target dbt RUNS, not the descriptor's: it falls back to the + // workspace warehouse's and to the project's own default, so reading the + // descriptor's would put two inherited targets under one empty name. + p.effective_target.as_deref(), + // The pair `relation_root` reports to the graph's drift check, taken + // apart so neither can absorb the other's delimiter below. + p.default_schema.as_deref(), + p.default_database.as_deref(), + ) +} + +/// The warehouse and the target name the environment; the database and schema +/// they resolve to are in the key because a repointed warehouse resource or a +/// moved schema keeps both names while putting the relations somewhere else — +/// and a manifest is a list of relation names, so a deferral has no other way to +/// notice. A move therefore reads as an environment nothing has published yet. +/// +/// Length-prefixed rather than joined on a separator. Every component but the +/// warehouse is spelled by the user — a dbt target name and a schema are both +/// arbitrary strings a profile may quote — so `prod|analytics` + `scratch` and +/// `prod` + `analytics|scratch` would otherwise be one key, and a profile moving +/// between them would read as the same environment rather than as one nothing +/// has published. Same reasoning as `stable_digest`, and still legible in a row: +/// `4:main|4:prod|9:analytics|12:dbt_wh_defer`. What a MESSAGE names is +/// `environment_label`, since this encoding is for storage. +fn environment_key( + warehouse: Option<&str>, + target: Option<&str>, + schema: Option<&str>, + database: Option<&str>, +) -> String { + [warehouse, target, schema, database] + .iter() + .map(|v| { + let v = v.unwrap_or(""); + format!("{}:{v}", v.len()) + }) + .collect::>() + .join("|") +} + +/// The environment as a message names it: the key above is length-prefixed for +/// storage, which is not something to put in front of a caller. +pub(crate) fn environment_label(p: &PreparedProject) -> String { + format!( + "warehouse `{}`, target `{}`, relations in `{}`", + p.warehouse.as_deref().unwrap_or("(none)"), + p.effective_target + .as_deref() + .unwrap_or("(the profile's default)"), + match (p.default_database.as_deref(), p.default_schema.as_deref()) { + (Some(db), Some(schema)) => format!("{db}.{schema}"), + (None, Some(schema)) => schema.to_string(), + _ => "(the adapter's default)".to_string(), + } + ) +} + +/// The state one environment last published. +pub(crate) struct StoredState { + pub manifest: String, + pub run_results: Option, + /// The run that published it, so a deferring run can say what it deferred to. + pub job_id: Uuid, +} + +/// Publish this run's artifacts as the environment's state. +/// +/// Called for a run that BUILT what the script's own descriptor selects and +/// succeeded (see `handle_dbt_job`). Best-effort in the same sense as the retry +/// state: losing it costs the next deferral, not the run that just finished. +/// +/// **What the artifacts may carry follows from that condition.** A publishing run +/// added nothing of its own — no `select` or `vars` override, and a descriptor +/// interpolating a `{{ }}` placeholder into `vars` never publishes at all — so +/// dbt's `run_results.json` records the descriptor's own arguments, which are the +/// script's content. That is why this is keyed by environment where +/// `dbt_run_state` is keyed by principal: the retry state holds whatever a caller +/// submitted, this holds what the script says. Widen the publish condition and +/// that stops being true. +pub(crate) async fn publish( + p: &PreparedProject, + w_id: &str, + job_id: &Uuid, + // The version this job ran. `None` for a preview, which publishes nothing. + script_hash: Option, + // A build recovered by the automatic in-job node retry has a + // `run_results.json` naming only the nodes that retry redid. The manifest is + // unaffected — it is a function of the project, not of what ran — so the + // state is published without results rather than with a set describing some + // other slice of the build. + results_are_partial: bool, + conn: &Connection, +) -> error::Result<()> { + let Connection::Sql(db) = conn else { + // An agent worker reaches the database only through the API, which does + // not expose this table. + return Ok(()); + }; + if p.script_path.is_empty() { + // A preview has no path to key state on, and an empty one would be + // shared by every dbt script in the workspace. + return Ok(()); + } + if p.templated_location { + // Refused on this side too, not only where a deferral reads. A template + // renders to one location per environment while the key sees the + // template, so publishing would file this run's manifest under a key a + // literal profile shares — and de-templating later would make that stale + // manifest readable as the new location's. + return Ok(()); + } + let artifacts = p.project_dir.join(ARTIFACTS_DIR); + // The manifest is what a deferral resolves through, so there is no state + // without one. Every engine writes it beside the results of a build, so this + // is the invocation that built nothing rather than a case to report. + let Ok(manifest) = tokio::fs::read_to_string(artifacts.join("manifest.json")).await else { + return Ok(()); + }; + let run_results = match results_are_partial { + true => None, + false => tokio::fs::read_to_string(artifacts.join("run_results.json")) + .await + .ok(), + }; + let environment = environment(p); + // Uploaded BEFORE the transaction, and to this publication's own keys, so two + // publishers cannot collide on them and nothing here can overwrite an + // artifact a committed row still names. A failure below has only its own + // objects to drop. + let nonce = Uuid::new_v4(); + let (manifest, manifest_key) = store( + manifest, + "manifest.json", + &environment, + &p.script_path, + w_id, + job_id, + &nonce, + ) + .await?; + let (run_results, run_results_key) = match run_results { + Some(r) => match store( + r, + "run_results.json", + &environment, + &p.script_path, + w_id, + job_id, + &nonce, + ) + .await + { + Ok(stored) => stored, + Err(e) => { + forget_objects(&[manifest_key, None]).await; + return Err(e); + } + }, + None => (None, None), + }; + let mine = [manifest_key.clone(), run_results_key.clone()]; + // One publisher per environment at a time, so the row and the objects it + // displaces are settled by one of them at a time. An advisory lock rather + // than the row's, because the first publish of an environment has no row to + // lock and is exactly when two runs of a newly deployed script are most + // likely to race. + let mut tx = match db.begin().await { + Ok(tx) => tx, + Err(e) => { + forget_objects(&mine).await; + return Err(e.into()); + } + }; + let staged = async { + sqlx::query_scalar!( + "SELECT pg_advisory_xact_lock($1)", + publication_lock(w_id, &p.script_path, &environment) + ) + .execute(&mut *tx) + .await?; + // The script row FIRST, and held, so a rename, archive or delete of this + // path either waits for this publication or is seen by it. Reading it + // unlocked leaves a window where lifecycle cleanup finds no row to clear, + // finishes, and this transaction then commits state at a path a new + // script goes on to occupy. Script row before sidecar is also the order + // every other dbt writer takes, which is what keeps the two off a + // deadlock. + // + // The version, not just the path: "some live dbt script is here" is also + // satisfied by a script created at a path this one was renamed away from. + // A preview names no version, so `script_hash` is NULL and nothing + // matches — right for a run of content that was never deployed. + let owns_path = sqlx::query_scalar!( + "SELECT 1 FROM script + WHERE workspace_id = $1 AND path = $2 + AND deleted = false AND archived = false AND language = 'dbt' + AND (hash = $3 OR $3 = ANY(parent_hashes)) + FOR SHARE", + w_id, + &p.script_path, + script_hash, + ) + .fetch_optional(&mut *tx) + .await? + .is_some(); + if !owns_path { + return error::Result::Ok(None); + } + // What the row points at NOW, so those objects can go once this one is + // committed in their place — never before, since a reader that has + // already read the row is about to fetch them. + let displaced = sqlx::query!( + "SELECT manifest_key, run_results_key FROM dbt_environment_state + WHERE workspace_id = $1 AND script_path = $2 AND environment = $3", + w_id, + &p.script_path, + environment + ) + .fetch_optional(&mut *tx) + .await? + .map(|r| [r.manifest_key, r.run_results_key]) + .unwrap_or_default() + // Never a key this publication is about to commit. The keys carry a + // per-execution nonce so the two cannot coincide, and this is what says + // so rather than leaving it to be re-derived. + .map(|k| k.filter(|k| !mine.iter().flatten().any(|m| m == k))); + sqlx::query!( + "INSERT INTO dbt_environment_state (workspace_id, script_path, environment, job_id, + manifest, manifest_key, run_results, + run_results_key, updated_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, now()) + ON CONFLICT (workspace_id, script_path, environment) DO UPDATE SET + job_id = EXCLUDED.job_id, manifest = EXCLUDED.manifest, + manifest_key = EXCLUDED.manifest_key, run_results = EXCLUDED.run_results, + run_results_key = EXCLUDED.run_results_key, updated_at = now()", + w_id, + &p.script_path, + environment, + job_id, + manifest, + manifest_key, + run_results, + run_results_key, + ) + .execute(&mut *tx) + .await?; + error::Result::Ok(Some(displaced)) + } + .await; + let displaced = match staged { + // Refused by the guard, or the write failed: nothing is committed and + // what was uploaded above has no row naming it. + Ok(None) | Err(_) => { + forget_objects(&mine).await; + return staged.map(|_| ()); + } + Ok(Some(displaced)) => displaced, + }; + // A commit that reports an error may still have committed — what was lost can + // be the acknowledgement. Dropping this run's objects would then leave the + // committed row naming objects that are gone, and every deferral would fail + // until the next publication; an orphan costs storage instead. + tx.commit().await?; + forget_objects(&displaced).await; + Ok(()) +} + +/// The environment's state, or `None` where nothing has published one. +pub(crate) async fn load( + p: &PreparedProject, + w_id: &str, + conn: &Connection, +) -> error::Result> { + let Connection::Sql(db) = conn else { + return Err(Error::BadRequest( + "`defer` resolves a `ref()` through the dbt state stored for this environment, which \ + an agent worker cannot read: it reaches the database only through the API. Run this \ + script on a worker of the main group, or without `defer`" + .to_string(), + )); + }; + let environment = environment(p); + // A publication committing between the row and the objects it named drops + // those objects, so a miss is re-read rather than reported as a state that is + // not there. Re-read for as long as the row keeps MOVING: a reader takes no + // lock, so back-to-back publications can each overtake it, and a fixed one + // retry would report the second as missing. An unmoved row is the other + // answer — nothing republished, so the object really is gone. + let mut tried: Option<(Uuid, Option, Option)> = None; + for _ in 0..PUBLICATIONS_OUTRUN { + let Some(row) = sqlx::query!( + "SELECT job_id, manifest, manifest_key, run_results, run_results_key + FROM dbt_environment_state + WHERE workspace_id = $1 AND script_path = $2 AND environment = $3", + w_id, + &p.script_path, + environment + ) + .fetch_optional(db) + .await? + else { + return Ok(None); + }; + let seen = ( + row.job_id, + row.manifest_key.clone(), + row.run_results_key.clone(), + ); + let fetched = async { + let manifest = fetch(row.manifest, row.manifest_key).await?; + let run_results = fetch(row.run_results, row.run_results_key).await?; + error::Result::Ok((manifest, run_results)) + } + .await; + match fetched { + Ok((Some(manifest), run_results)) => { + return Ok(Some(StoredState { manifest, run_results, job_id: seen.0 })) + } + Ok((None, _)) => return Ok(None), + Err(e) => { + if tried.as_ref() == Some(&seen) { + return Err(e); + } + tried = Some(seen); + } + } + } + Err(Error::internal_err( + "the dbt state for this environment was replaced faster than it could be read; run this \ + script again" + .to_string(), + )) +} + +/// How many publications a read may lose to before it gives up. Each one costs a +/// re-read, and a project publishing this often while another run defers is +/// already contending for the same relations. +const PUBLICATIONS_OUTRUN: usize = 5; + +/// A `manifest.json` for a state directory, whichever side it comes from. +/// +/// One enum because the three restores — a deferral's stored state, a retry's +/// worker-local generation, a retry's database row — differ only in where the +/// bytes are, and a second copy of the directory layout is a second chance for +/// one of them to write a directory dbt reads differently. +pub(crate) enum StateManifest { + Bytes(String), + /// A file on this worker, copied rather than read into memory: a manifest + /// grows with the project. + CopyOf(PathBuf), + None, +} + +/// Write the artifacts a dbt state directory holds, creating it if needed. +/// +/// Returns whether a `manifest.json` ended up there — a worker-local generation +/// can be pruned out from under a restore, and the caller then owes a +/// `dbt parse` for one. +pub(crate) async fn write_state_dir( + dir: &Path, + run_results: Option<&str>, + manifest: StateManifest, +) -> error::Result { + tokio::fs::create_dir_all(dir) + .await + .map_err(|e| Error::internal_err(format!("preparing the dbt state directory: {e}")))?; + if let Some(run_results) = run_results { + tokio::fs::write(dir.join("run_results.json"), run_results) + .await + .map_err(|e| Error::internal_err(format!("writing run_results.json: {e}")))?; + } + Ok(match manifest { + StateManifest::Bytes(m) => { + tokio::fs::write(dir.join("manifest.json"), m) + .await + .map_err(|e| Error::internal_err(format!("writing manifest.json: {e}")))?; + true + } + StateManifest::CopyOf(from) => tokio::fs::copy(from, dir.join("manifest.json")) + .await + .is_ok(), + StateManifest::None => false, + }) +} + +/// The advisory lock one environment's publishers take, so only one of them +/// settles the row and the objects it displaces at a time. +/// +/// Derived from the same three components as the row's key. Two environments +/// whose digests collide wait for each other, which costs a moment and nothing +/// else. +fn publication_lock(w_id: &str, script_path: &str, environment: &str) -> i64 { + let d = digest(&format!("{w_id}|{script_path}|{environment}")); + // Parsed unsigned and reinterpreted: half of all digests set the top bit, + // and read as `i64` those overflow and would collapse onto one key. + u64::from_str_radix(&d[..16], 16).unwrap_or_default() as i64 +} + +/// The object-storage key an artifact takes. +/// +/// One key per PUBLICATION, so an upload never overwrites an artifact the +/// committed row still names: a run that fails between its two uploads, or +/// between them and its row, leaves the state pointing at the pair it already +/// had. The row switches to these in one statement and the objects it displaced +/// are dropped afterwards. The path and environment are only a prefix — the row +/// is what says where an artifact is, so state that moves with a renamed script +/// keeps naming objects under the old one. Digested because a Windmill path and a +/// schema name may both carry characters an object key gives meaning to. +/// +/// The `nonce` is per EXECUTION rather than per job, because zombie recovery +/// re-runs a job under its own id: keyed on that alone, the second attempt would +/// overwrite the objects the first attempt's committed row still names, and then +/// read those same keys back as displaced and drop them. +fn object_key( + w_id: &str, + script_path: &str, + environment: &str, + job_id: &Uuid, + nonce: &Uuid, + artifact: &str, +) -> String { + format!( + "wmill_dbt_state/{w_id}/{}/{job_id}.{nonce}/{artifact}", + digest(&format!("{script_path}|{environment}")) + ) +} + +/// Put an artifact where its size says it belongs: `(inline, key)`, exactly one +/// of which is set. +#[allow(clippy::too_many_arguments)] +async fn store( + value: String, + artifact: &str, + environment: &str, + script_path: &str, + w_id: &str, + job_id: &Uuid, + nonce: &Uuid, +) -> error::Result<(Option, Option)> { + if value.len() <= *DBT_STATE_INLINE_MAX_BYTES { + return Ok((Some(value), None)); + } + let key = object_key(w_id, script_path, environment, job_id, nonce, artifact); + let size = value.len(); + if put_object(&key, value).await? { + return Ok((None, Some(key))); + } + Err(Error::BadRequest(format!( + "this project's {artifact} is {}, past the {} this instance keeps in the database, and \ + this instance has no object storage configured to hold it. Configure instance object \ + storage, or raise DBT_STATE_INLINE_MAX_BYTES", + mib(size), + mib(*DBT_STATE_INLINE_MAX_BYTES), + ))) +} + +fn mib(bytes: usize) -> String { + format!("{:.1} MiB", bytes as f64 / (1024.0 * 1024.0)) +} + +/// Read an artifact back from whichever home the row names. +async fn fetch(inline: Option, key: Option) -> error::Result> { + match (inline, key) { + (Some(inline), _) => Ok(Some(inline)), + (None, Some(key)) => get_object(&key).await.map(Some), + (None, None) => Ok(None), + } +} + +/// Drop the objects nothing points at any more. Best-effort: an object left +/// behind costs storage, and there is nothing useful to do about it in the path +/// of a run that has already finished. +async fn forget_objects(keys: &[Option; 2]) { + for key in keys.iter().flatten() { + delete_object(key).await; + } +} + +/// Whether the artifact was stored. `false` means this instance has no object +/// storage to put it in. +/// +/// The INSTANCE store, where every other internal worker artifact lives — bun +/// bundles, python wheels, job logs, the global cache. Not the workspace's: +/// that bucket is the one workspace members read and write through +/// `job_helpers/*` and `wmill.write_s3_file`, so a manifest there is one any +/// member could replace, and the next deferring run would hand dbt an +/// attacker-chosen `defer_relation` for every unbuilt `ref()` while holding the +/// script's warehouse credentials. Its compiled SQL would be readable there too, +/// for a project the reader may have no access to. +#[cfg(all(feature = "enterprise", feature = "parquet"))] +async fn put_object(key: &str, value: String) -> error::Result { + use windmill_object_store::object_store_reexports::Path as ObjectPath; + let Some(store) = windmill_object_store::get_object_store().await else { + return Ok(false); + }; + store + .put(&ObjectPath::from(key), bytes::Bytes::from(value).into()) + .await + .map_err(|e| Error::internal_err(format!("storing the dbt state at {key}: {e:#}")))?; + Ok(true) +} + +#[cfg(all(feature = "enterprise", feature = "parquet"))] +async fn get_object(key: &str) -> error::Result { + let Some(store) = windmill_object_store::get_object_store().await else { + return Err(missing_storage()); + }; + let bytes = windmill_object_store::attempt_fetch_bytes(store, key).await?; + String::from_utf8(bytes.to_vec()) + .map_err(|e| Error::internal_err(format!("the stored dbt state is not valid UTF-8: {e}"))) +} + +#[cfg(all(feature = "enterprise", feature = "parquet"))] +async fn delete_object(key: &str) { + use windmill_object_store::object_store_reexports::Path as ObjectPath; + let Some(store) = windmill_object_store::get_object_store().await else { + return; + }; + if let Err(e) = store.delete(&ObjectPath::from(key)).await { + tracing::warn!("dbt: could not drop the superseded state object {key}: {e:#}"); + } +} + +#[cfg(not(all(feature = "enterprise", feature = "parquet")))] +async fn delete_object(_key: &str) {} + +/// A build without the instance store carries no client at all, so an oversized +/// artifact has nowhere but the row, and a row naming a key was written by a +/// worker that did have one. +#[cfg(not(all(feature = "enterprise", feature = "parquet")))] +async fn put_object(_key: &str, _value: String) -> error::Result { + Ok(false) +} + +#[cfg(not(all(feature = "enterprise", feature = "parquet")))] +async fn get_object(_key: &str) -> error::Result { + Err(missing_storage()) +} + +fn missing_storage() -> Error { + Error::BadRequest( + "the dbt state for this environment is in the instance's object storage, which this \ + worker cannot reach: it is no longer configured, or this worker was built without \ + object-storage support" + .to_string(), + ) +} + +/// The stored state this run resolves its unbuilt `ref()`s through, materialised +/// into the job directory at `STATE_DIR`. +#[derive(Clone, Debug)] +pub(crate) struct Deferral { + /// The run that published the state, so the job log and the result can say + /// what this one deferred to. + pub published_by: Uuid, + /// Whether the state carries `run_results.json` beside its manifest. A build + /// recovered by node retry publishes without one, and that is the only file a + /// `result:` selector reads. + pub has_run_results: bool, +} + +/// Materialise the environment's state so `--state` has a directory to read. +/// +/// Refused rather than run without deferral where nothing is published: the run +/// would build against a `ref()` resolving into the schema it writes, and fail +/// deep inside dbt with a relation-not-found the caller has no way to connect +/// back to a missing state. +pub(crate) async fn prepare_deferral( + p: &PreparedProject, + w_id: &str, + job_dir: &str, + conn: &Connection, +) -> error::Result { + if p.script_path.is_empty() { + return Err(Error::BadRequest( + "`defer` resolves a `ref()` through the state a previous run of this script \ + published, so it needs a deployed script; a preview run has no environment to have \ + published one" + .to_string(), + )); + } + // An environment is the warehouse, the target and where they RESOLVE to, and + // a `profiles.yml` that templates its schema or database resolves somewhere + // this runtime does not render. Two renderings would then share one + // environment, and a deferral after the value changed would resolve every + // unbuilt `ref()` through the previous location's manifest. + if p.templated_location { + return Err(Error::BadRequest( + "this project's profile selects its schema or database with a template, which dbt \ + renders and Windmill does not — so two environments cannot be told apart and a \ + deferral could resolve through the wrong one's manifest. Spell the target's schema \ + and database literally to use `defer`" + .to_string(), + )); + } + let Some(state) = load(p, w_id, conn).await? else { + return Err(Error::BadRequest(format!( + "no dbt state is stored for this environment ({}), so a `ref()` this run does not \ + build has no relation to resolve to. It is published by a successful run that adds \ + nothing of its own: one overriding `select` or `vars` does not publish, and neither \ + does any run of a descriptor that interpolates a `{{{{ }}}}` placeholder into `vars` \ + or a `$var:` into `env` — those describe a model set the caller's arguments decided. \ + Run this script once without `defer` and without overrides", + environment_label(p) + ))); + }; + let has_run_results = state.run_results.is_some(); + write_state_dir( + &PathBuf::from(job_dir).join(STATE_DIR), + state.run_results.as_deref(), + StateManifest::Bytes(state.manifest), + ) + .await?; + Ok(Deferral { published_by: state.job_id, has_run_results }) +} + +#[cfg(test)] +mod tests { + use super::*; + + // Every component earns its place: a deferral resolves relation NAMES, so + // state published where those names meant something else has to read as no + // state at all rather than as state that silently no longer fits. + #[test] + fn a_moved_profile_is_another_environment() { + let here = environment_key(Some("main"), Some("prod"), Some("analytics"), Some("wh")); + assert_eq!( + here, + environment_key(Some("main"), Some("prod"), Some("analytics"), Some("wh")) + ); + assert_ne!( + here, + environment_key(Some("other"), Some("prod"), Some("analytics"), Some("wh")) + ); + assert_ne!( + here, + environment_key(Some("main"), Some("dev"), Some("analytics"), Some("wh")) + ); + assert_ne!( + here, + environment_key(Some("main"), Some("prod"), Some("marts"), Some("wh")) + ); + assert_ne!( + here, + environment_key( + Some("main"), + Some("prod"), + Some("analytics"), + Some("other_db") + ) + ); + } + + // A target name and a schema are both the user's own strings, so a component + // carrying the separator must not be able to spell another tuple's key: a + // profile moving between the two would read as the same environment and + // defer through the manifest of relations that are somewhere else. + #[test] + fn a_component_cannot_spell_another_environments_key() { + assert_ne!( + environment_key(Some("main"), Some("prod|analytics"), Some("scratch"), None), + environment_key(Some("main"), Some("prod"), Some("analytics|scratch"), None) + ); + assert_ne!( + environment_key(Some("main"), Some("prod"), Some("a"), Some("b|c")), + environment_key(Some("main"), Some("prod"), Some("a|b"), Some("c")) + ); + // A component the profile leaves out is the same environment as one it + // spells empty: there is no target named "". + assert_eq!( + environment_key(Some("main"), None, Some("a"), None), + environment_key(Some("main"), Some(""), Some("a"), Some("")) + ); + } + + // Two environments must not queue behind one advisory lock, which is what a + // digest folded through a signed parse did for every one whose top bit is + // set — half of them. + #[test] + fn each_environment_gets_its_own_publication_lock() { + let mut seen = std::collections::HashSet::new(); + for i in 0..64 { + seen.insert(publication_lock( + "ws", + "f/a/p", + &format!("main|prod|s{i}|db"), + )); + } + assert_eq!(seen.len(), 64); + assert_eq!( + publication_lock("ws", "f/a/p", "e"), + publication_lock("ws", "f/a/p", "e") + ); + } +} diff --git a/backend/windmill-worker/src/lib.rs b/backend/windmill-worker/src/lib.rs index e3e7868548..8aa739ecd8 100644 --- a/backend/windmill-worker/src/lib.rs +++ b/backend/windmill-worker/src/lib.rs @@ -36,6 +36,7 @@ mod csharp_executor; mod dbt_engine; mod dbt_executor; mod dbt_profiles; +mod dbt_state; #[cfg(feature = "private")] mod dedicated_worker_ee; mod dedicated_worker_oss; diff --git a/docs/dbt-runtime.md b/docs/dbt-runtime.md index 3749bda3ea..acb4621b27 100644 --- a/docs/dbt-runtime.md +++ b/docs/dbt-runtime.md @@ -13,8 +13,8 @@ the dominant way dbt is orchestrated today. - **In**: run an unmodified dbt project synced into Windmill, one Windmill job per invocation, live per-model observability, dbt models as first-class assets in the existing asset graph. -- **Out**: one Windmill job per dbt model, `state:modified` / slim CI, - `dbt docs` hosting, semantic layer, dbt platform integration. +- **Out**: one Windmill job per dbt model, slim CI orchestration, `dbt docs` + hosting, semantic layer, dbt platform integration. - **CE**: the runtime, the manifest ingest, the asset graph and every piece of UI ship in CE, as do all adapters except two. Only the `mssql` and `oracle` adapters are EE, mirroring the native `ScriptLang` boundary (decision 21). @@ -35,7 +35,7 @@ the dominant way dbt is orchestrated today. | 10 | Private repo auth | Not applicable: the project is synced, not fetched | | 11 | Asset kind | `dbt:////` — keyed on the relation, not on dbt's node id. See below | | 12 | Graph refresh | Deploy-time, re-ingested per run only when the descriptor is dynamic, plus an explicit `parse` of the editor's buffer. See below | -| 13 | Manifest storage | Sidecar table for nodes/edges. Full manifest **not** stored — see below | +| 13 | Manifest storage | Sidecar table for nodes/edges; the whole manifest is kept once per environment, for deferral — see below | | 14 | Metadata depth | Tests, strategy, tags, freshness, column descriptions. Column **lineage** is not in the manifest — see below | | 15 | Node rendering | Asset nodes per model plus one runnable node for the script | | 16 | Progress | Live, from the JSON event stream | @@ -48,6 +48,7 @@ the dominant way dbt is orchestrated today. | 23 | Descriptor | `wm_dbt.yaml` inside the project, OPTIONAL. See below | | 24 | Warehouse | Configured on the workspace by name, `main` by default. See below | | 25 | Cascade direction | Into a relation, not out of a run: `// materialize manual dbt://…` declares a write from any language but dbt's own and wakes `# on dbt://…` subscribers; a finished dbt run still does not dispatch. See "No cascade *from* dbt" | +| 26 | Deferral | A durable state per environment, published by the runs whose relations are the script's; `defer` is a per-run toggle. See below | ## Decision 1: engine toggle, and why the shipped default is not Fusion yet @@ -1228,13 +1229,343 @@ block, since that is what `dbt_run_state` saves and `invocation_args` publishes. without failing. Overriding this would make the same project behave differently on Windmill than locally, breaking the core promise. +## Durable state per environment, and what defers to it + +`dbt --defer --state ` resolves a `ref()` the run does not build to the +relation the manifest in `` names, instead of to the schema this run writes +into. That is what lets one model be rebuilt into a scratch schema without +rebuilding everything above it, and it needs a manifest of the environment the +project actually lives in. + +Nothing that already existed could supply one. `dbt_run_state` answers a +different question — it holds the LAST run whatever its outcome, keyed by the +principal, so `dbt retry` can resume its failures — and the worker-local +generations behind it are a cache: the next run of a project usually lands on a +worker holding neither artifact. So the state is its own table, +`dbt_environment_state`, one row per (workspace, script path, environment), +holding `manifest.json` and `run_results.json` from the last SUCCESSFUL run. +Success is half of the contract: a relation a later run defers to has to exist. + +### The environment is the warehouse, the target and where they resolve to + +The workspace warehouse's name, the target dbt actually runs, and the database +and schema that target resolves to — the pair `relation_root` reports to the +graph's drift check. Each component is length-prefixed rather than joined on a +separator — `|||`, each written `:`, +so `main`/`prod`/`analytics`/`dbt_wh_defer` is stored as +`4:main|4:prod|9:analytics|12:dbt_wh_defer`. A target name and a schema are both +the user's own strings, so `prod|analytics` + `scratch` and `prod` + +`analytics|scratch` would otherwise be one key, and a profile moving between them +would read as the same environment rather than as one nothing has published. What +a message names is spelled out instead, never the encoded key. + +The target is the EFFECTIVE one, not the descriptor's `profile.target`: a +descriptor naming none inherits the workspace warehouse's, or the default in the +project's own `profiles.yml`, so reading the descriptor's would file every +inherited target under one empty name — and a `target.name` macro decides where a +model is built. + +The last two are in the key because deferring is resolving a relation NAME. A +warehouse repointed at another database, or a `profile.schema` moved by a +redeploy, keeps the first two while putting every relation somewhere else, and a +manifest is a list of relation names — there is no other way to notice. Keyed on +the first two alone, such a move would hand the next deferring run the names of +relations that are no longer there. Keyed on all four, it reads as an +environment nothing has published yet, which is what it is. + +What the key deliberately does NOT carry is the resolved connection. That is the +`profile_digest` a retry is held to, and it moves when a password is rotated, +which moves no relation; a warehouse pointing somewhere else entirely is +decision 11's accepted limitation, spelled the same way here as everywhere else. + +Today one script has one environment, because a descriptor fixes both the +warehouse and the target and a run cannot override either. The key is what makes +the *later* item — fork and preview environments — an addition rather than a +migration, and what makes a profile move detectable now. + +### Which runs publish it + +A successful `build` that did not itself defer, and whose graph becomes what the +script owns (`GraphRefresh::publishes_ownership`) — the same condition as the +graph's and the same reason: an invocation that scoped its own model set — a +`vars` or `select` override, or a descriptor dynamic by construction — describes +where the CALLER put those relations, not where this project's models live. +Publishing it would point every later deferral at one caller's scratch schema. + +**A run that deferred never publishes, whatever narrowed it**, and that is a +separate condition rather than a consequence of the first. A deferring run built +some of the relations its manifest names and resolved the rest out of the state +it read, so recording that manifest would claim relations nothing built — and a +model renamed since would be recorded under a name only a full build creates, +breaking every later deferral until one repairs it. `publishes_ownership` cannot +see this: it reads the caller's overrides, and a descriptor that already narrows +`select` needs none. + +A `retry` publishes nothing. Its `run_results.json` names only the nodes it +redid, so the environment would come to claim a run of a handful of models. The +environment's state is therefore the last full successful build, exactly as dbt +Cloud's "last successful run" is, and a run recovered by a retry leaves it at +the previous one. + +The AUTOMATIC in-job node retry is the same artifact under a different name: a +build it recovers is a successful build, but the `run_results.json` on disk is +the retry's. Such a run publishes the manifest **without** results, rather than +with a set describing some other slice of the build — the manifest is a function +of the project rather than of what ran, so deferral is unaffected. A `result:` +selector is the one thing left with nothing to read, and it is refused by name +against such a publication rather than passed to dbt (see "Selectors that read +the state" below). + +Under `test_behavior: after_all` the stored `run_results.json` is the test +phase's, because that is what the second invocation leaves in the target +directory — the same artifact a local `dbt run && dbt test` leaves behind. + +**What that condition means for what the artifacts may carry**, and why this +table is keyed by environment where `dbt_run_state` is keyed by principal. dbt +records the invocation's flags into `run_results.json`, and Windmill resolves +`$var:` / `$res:` references before dbt sees them — which is exactly why the +retry state is per-principal, so one caller's resolved `select` and `vars` are +not restorable by the next. Here they cannot be one caller's: a publishing run +added nothing of its own, and a descriptor that interpolates a `{{ }}` +placeholder into `vars` never publishes at all, so what is recorded is the +descriptor's own arguments — the script's content, which anyone entitled to run +it may already read. Widen the publish condition and that stops being true. + +### Where the blob goes + +`run_results.json` is small; `manifest.json` is not, and grows with the project +(535 KB on a two-model fixture). Each takes the same two homes: inline in the row +under `DBT_STATE_INLINE_MAX_BYTES` (8 MiB), and the INSTANCE's object storage +above it, with the row keeping the key. Inline is what makes the feature work on +an instance that has configured no storage at all; the ceiling is what stops one +project's manifest from becoming a multi-megabyte row rewritten by every run. A +project past the ceiling with no storage configured is told so, in the job log, +naming the setting and the variable — the run itself still succeeds, since +losing the state costs the next deferral rather than the build that just ran. + +**The instance store, not the workspace's**, which is where every other internal +worker artifact already lives (bun bundles, python wheels, job logs, the global +cache). The workspace bucket is the one members read and write through +`job_helpers/*` and `wmill.write_s3_file` with a caller-supplied key, and only +`volumes/` is reserved there — so a manifest under it is one any member could +replace, and the next deferring run would hand dbt an attacker-chosen +`defer_relation` for every unbuilt `ref()` while holding the script's warehouse +credentials. Its compiled SQL would be readable there too, for a project the +reader may have no access to. The consequence to know: a project past the ceiling +needs the instance store configured, which is an EE feature, so on CE the ceiling +is the limit and `DBT_STATE_INLINE_MAX_BYTES` is how it moves. + +Each publication writes its OWN keys +(`wmill_dbt_state///./`) +and the row switches to them in one statement, so an upload never overwrites an +artifact the committed row still names: a run that fails between its two uploads, +or between them and its row, leaves the state pointing at the pair it already +had. The objects the commit displaced are dropped afterwards, never before, since +a reader that has already read the row is about to fetch them; a reader that +loses that race re-reads for as long as the row keeps MOVING, rather than +reporting a state that is there. A reader takes no lock, so successive +publications can each overtake one; an unmoved row whose objects are gone is the +error that means what it says, and a bound on the re-reads is the other, for a +project republishing faster than a run can read. What a publication uploaded and then could not commit is dropped on the +way out — except after a commit that REPORTED an error, where what was lost may +be only the acknowledgement: dropping then would leave a committed row naming +objects that are gone, so an orphan is the cheaper side to take. + +The path and the environment are only a prefix of that key. The row is what says +where an artifact is, which is why state can travel with a renamed script and go +on naming objects under the old path's digest. The rest of the key is the job and +a per-EXECUTION nonce — zombie recovery re-runs a job under its own id, so keyed +on that alone a second attempt would overwrite the objects the first attempt's +committed row still names, then read those keys back as displaced and drop them. + +Publishers of one environment serialize on `pg_advisory_xact_lock`, so only one +of them settles the row and the objects it displaces at a time — an advisory lock +rather than the row's, because the first publish of an environment has no row to +lock and is exactly when two runs of a newly deployed script are most likely to +race. + +### Retention + +None, deliberately, and this is where it differs from the graph tables next +door. Those are pruned by age by the dbt runs themselves because their reader is +a transient run page. This one holds a single row per script per environment, +replaced in place, so it does not grow with runs — and its reader is every later +run of that script, so a project that runs monthly must still find last month's +state. It goes with the script instead: a path no live dbt version occupies any +more clears it, alongside `dbt_run_state` (`clear_dbt_script_state`, +`clear_dbt_script_state_if_path_retired`). + +The write carries a guard of its own, and it names the VERSION rather than the +path: the live dbt script there must be the one this job ran, or a later version +of it (`hash = $n OR $n = ANY(parent_hashes)`). "Some live dbt script is here" — +which is what the retry state settles for — is also satisfied by a script created +at a path this one was renamed away from, and this job's manifest would then +become that project's deferral state. A preview names no version and so publishes +nothing, which is right for a run of content that was never deployed. + +The job's KIND is checked beside it, because a preview carries a caller-supplied +`script_hash` into `runnable_id` (`run_preview_script`): the version alone would +let anyone who may run a job publish arbitrary content as a deployed script's +state. A flow or app step naming a deployed dbt script by path is an ordinary +`script` job carrying that script's own hash, so it publishes like any other run; +only INLINE flow code is a `FlowScript`, and that has no deployed version to +publish for. + +That guard HOLDS the script row (`FOR SHARE`) for the rest of the publication, so +a rename, archive or delete of the path either waits for it or is seen by it. +Read unlocked, it leaves a window where the lifecycle clear finds no row to take, +finishes, and the publication then commits state at a path a new script goes on +to occupy. The script row is taken before the sidecar, which is the order every +other dbt writer takes and what keeps the two off a deadlock. + +An artifact too large for its row is left in the store when the row is cleared, +as a deleted script leaves its bundle: reaching it from the delete would mean an +object-store client in `windmill-common` and a delete that has to land after the +caller's transaction commits, for one object per environment of a script that is +gone. + +### Asking for it + +`defer` is a field on the `build` command block, defaulting to the descriptor's +own `defer:`. A per-run toggle rather than a descriptor-only setting, because the +run that publishes an environment's state and the run that defers to it are two +invocations of ONE script (decision 6: N scripts means N projects): a project +that could only defer by descriptor could never populate the state it reads. + +A project whose profile selects its schema or database with a TEMPLATE — either +delimiter, since dbt renders `{% … %}` blocks as well as `{{ … }}` — is refused a +deferral outright, and publishes no state either: dbt renders those and Windmill +does not, so two renderings resolve to one `relation_root`, and a +deferral after the value changed would resolve every unbuilt `ref()` through the +previous location's manifest. Both sides, because a published template would sit +under a key a literal profile shares, and de-templating later would make that +stale manifest readable as the new location's. It covers a project-owned +`profiles.yml`, a `dbt_profile` resource — one block of the user's own file, +copied through unchanged — and a `profile.schema` written as given. Plainly +absent is different: that is the adapter's default, which does not move. + +A run that asks to defer with nothing published is refused, naming the +environment and the runs that cannot publish one. The alternative — running +without deferral — fails deep inside dbt with a relation-not-found the caller has +no way to connect back to a missing state. An agent worker is refused the same +way and for a reason it can act on: it reaches the database only through the API, +which does not expose this table. + +A `show` defers too, and every engine takes the flags on it. It compiles the +model it previews, so a model whose upstream this environment built and this run +did not is exactly the case a deferral exists for. So does the `dbt ls` that +resolves what a run's selection owns, without which a `result:` selector — which +reads `run_results.json` out of the state directory, and which `select` passes to +dbt verbatim — would fail before the build that would have honoured it. + +The result carries `deferred_to`, the run whose state was used. Without it what +a deferring run built against is unrecoverable, since the next successful run of +that environment replaces the state. + +### Selectors that read the state, and why they are refused rather than passed + +`--state` also feeds dbt's own selector methods, so publishing the state is what +makes `state:modified+`, `state:new` and `result:error+` resolve at all. Only a +deferring run is handed the directory, so a `state:` or `result:` method in +`select` or `exclude` without `defer` is refused before dbt starts. + +Refused, rather than left to dbt, because the engines disagree about it and two +of the three disagree silently. Given a state selector and no `--state`, +dbt-core 1.x raises (`Got a state selector method, but no comparison manifest`, +exit 2), but dbt-sa-cli 2.x and fusion read a MISSING state as an EMPTY one and +exit 0: `state:modified` then selects nothing and the run reports success having +built nothing, while `state:new` selects everything, because against an empty +state every node is new. A scheduled run that quietly stops doing work, or +quietly rebuilds the project, is the failure this state exists to prevent. + +From the DESCRIPTOR they are refused whether or not the run defers, and the +message says so. That selection is also what decides which nodes the script owns, +and the deploy resolves it before any run exists, with no state to compare +against. "Whatever changed last" is not an ownership answer. They describe one +run, so they belong in a run's own `select`. + +`source_status:` is refused under any setting: it compares `sources.json`, which +`dbt source freshness` writes and no run publishes here, so there is nothing to +compare against even while deferring. + +Two more refusals follow from the same argument, that a selector with nothing to +read must say so rather than resolve to a silent answer: + +- A `result:` method while deferring to a state that carries **no** + `run_results.json`. Publishing that is deliberate — a build recovered by + automatic node retry stores the manifest alone, its results describing the + retried nodes rather than the build ("Which runs publish it") — so `defer` + being on is not enough to know the file is there. Answerable only once the + state is loaded, so it is checked right after, naming the run that published. +- Any of them on a `parse`. A parse resolves a selection to store the graph and + never defers, so `defer` would not hand it a state at any setting, and the + remedy the other refusal offers would lead nowhere. It says that instead. + +Matching nothing is then an ordinary outcome for these methods, and for no +others. `state:modified+` selects the empty set exactly when nothing changed +since the published state, which is the answer a CI run wants, so a selection +naming a `state:` or `result:` method may resolve to no nodes. Such a run scoped +its own selection, so what it stores is a snapshot of its own and never what the +script owns, and nothing is un-wired by the empty set. + +The exemption is by METHOD, not by who chose the selection. Exempting every +caller-chosen one would take a misspelled model name, which resolves to nothing +just as surely, and report it as a build that did its work. An ordinary selection +matching nothing stays refused, from a run as from the descriptor — from the +descriptor because that one also decides ownership. + +Only what `select` and `exclude` spell directly. A method reached through a +`selectors.yml` definition is named nowhere the worker reads, and dbt's own +behaviour — including the silent one — is what stands there. + +### `--state` is also a retry's own argument, and that is a trap + +`dbt retry` reads the run it RESUMES from `--state`. Handed the deferral's +directory it resumes the successful run stored there, finds nothing failed, and +reports a green retry having rebuilt nothing — silently, on dbt-core 1.x, which +warns and exits 0. + +dbt-core 1.x has `--defer-state`, the deferral-only half of the pair, so a retry +there passes that and leaves `--state` alone. The Rust engines do not have it, +and a run that deferred is refused a retry on them, before the build: the +alternative is rebuilding the failed nodes with every `ref()` resolving into the +schema this run writes into, which for the narrowed run a deferral exists to +serve means writing them somewhere they do not belong. The automatic in-job node +retry is dropped for the same reason and says so in the log. + +The state directory is passed RELATIVE (`wm_dbt_state`, beside `wm_target` in the +job directory). dbt records the invocation's flags into `run_results.json` and a +later `dbt retry` restores them, so an absolute path would name the job directory +of the run being resumed, which is gone by then. Relative, it resolves against +the project root — whichever job directory the retry landed in. + +Three engine facts found while wiring this up, all worth knowing before filing a +bug against the feature. `dbt retry` on dbt-core 2.x restores **neither** the +resumed invocation's `--vars` nor its deferral: it re-parses with the current +(empty) ones, so a retry of a run that overrode `vars` rebuilds into the +descriptor's schema rather than the run's. That is independent of deferral and +predates it; the refusal above stops the deferring case from being the way it is +discovered. `dbt show` on either Rust engine prints a bare JSON array where +dbt-core frames it as `{"node": …, "show": […]}`, which `run_show` is written +against — so a preview there fails to parse whether or not it defers, and the +deferral itself resolves correctly under it. And neither Rust engine reached +dbt's own service-backed State (`--manage-state`) on any run measured here, so no +flag is passed to disable it. + +Because `select` reaches dbt verbatim, a deferring run also has a `--state` +directory for `result:` selectors, which is why `run_results.json` is stored +beside the manifest rather than the manifest alone. + ## Two decisions the implementation narrowed -**Decision 13 — no S3 copy of the manifest.** The sidecar holds every field the -graph renders; nothing reads a stored `manifest.json`, so writing one to S3 -would be an unread copy of data that is already reproducible by redeploying (or, -for a dynamic descriptor, by the next run). Worth adding the day something needs the -parts the sidecar drops — compiled SQL, macro definitions — and not before. +**Decision 13 — the manifest is stored once per environment, not per version.** +The sidecar holds every field the graph renders, so a copy of `manifest.json` +bought the graph nothing: it is reproducible by redeploying, or for a dynamic +descriptor by the next run. Deferral is the reader that changed that — it +resolves an unbuilt `ref()` through a manifest, and one on worker-local disk +answers for a machine's history rather than for the environment. So exactly one +manifest is kept per (script, environment), replaced by each successful run, +rather than one per version (see "Durable state per environment" above). **Decision 14 — column lineage is not available.** The decision assumed `manifest.json` carries column-to-column edges; it does not, in either core @@ -1280,7 +1611,10 @@ render through the existing `RunnableNode.svelte` / `AssetNode.svelte` / on the canvas mid-run. `record_materialization` per model. Profile and select pickers in the editor. Per-model failure triage in the run view. -**Phase 4 (not in this PR).** `--defer` and `state:modified`. Partition and +**Phase 4 (not in this PR).** Slim CI: the fork and preview environments a +deferral would name instead of its own. The selectors themselves are here, since +`state:` and `result:` read the published state like any deferral does; what is +missing is a per-branch environment to compare a CI run against. Partition and backfill integration so `BackfillRangeDialog.svelte` works on dbt models. `wmill dbt import ` reading `DbtDag(...)` kwargs. @@ -1317,6 +1651,16 @@ Against a real dbt project (jaffle_shop shape) and the local Postgres: own `profiles.yml` with env-var injection. 11. **Caching**: a second run reuses the cached `dbt_packages/` with no network fetch. +12. **Deferral**: a full run publishes the environment's state; a second run + that builds one downstream model into another schema resolves its unbuilt + `ref()` to the relation the state names, where the same run without `defer` + fails with relation-not-found. +13. **State selectors**: with a state published, `state:modified+` selects + nothing while the project is unchanged and exactly the changed model and its + children after one is edited. Without `defer` it is refused rather than + passed, and a `result:` selector against a state published by a + node-retry-recovered build is refused too, that one carrying no + `run_results.json`. Keep only tests that pin behavior a future change could break. Per AGENTS.md, delete development scaffolding before marking the PR ready. diff --git a/frontend/src/lib/script_helpers.ts b/frontend/src/lib/script_helpers.ts index 76ecfa59d8..c253bc64e7 100644 --- a/frontend/src/lib/script_helpers.ts +++ b/frontend/src/lib/script_helpers.ts @@ -1370,6 +1370,11 @@ test_behavior: build vars: {} threads: 4 full_refresh: false +# Resolve a ref() this run does not build through the state the last successful +# run of this environment published, instead of through the schema it writes +# into. The default for the run form's toggle: the run that publishes the state +# and the run that defers to it are two invocations of this one script. +defer: false # Rebuild the nodes a failed build left failed or skipped, in this same job, # before reporting failure. dbt confines a failure to its own subtree, so a # transient warehouse error costs those nodes rather than the whole project. From 0139467b01b82e4b3d474ca3f205358fa607d19a Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 8 Sep 2026 07:58:38 +0000 Subject: [PATCH 018/144] feat: ingest dbt column lineage and real column schemas from the engine's parquet index (#10977) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: column-level lineage for dbt from the engine's parquet index `manifest.json` carries no column-to-column edges, which is why decision 14 recorded column lineage as unavailable. The edges live in a different artifact: `dbt compile --static-analysis strict --write-index` writes `target/index/`, whose `dbt.column_lineage.parquet` holds them and whose `dbt.node_columns.parquet` holds every column of every node, typed and ordered rather than only the ones an author documented. Strict analysis rejects SQL the default accepts, so this is a separate compile with its own `--target-path`, opt-in per project via `column_lineage: true`, and best-effort throughout: a project it cannot analyze keeps exactly the graph it had, with the engine's own diagnostics in the job log. Storage mirrors `dbt_edge`: `dbt_column_edge` keyed by (path, version, job) with the same composite FK to `script` and the same sweeps. The typed column list lands in `dbt_node.column_schema`, beside `columns` rather than merged into it, so `columns` stays what the author declared. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01PRtsPQ3Ck69Fu9DNr7bMcJ * fix: address review findings on the dbt column-lineage pass - The workspace fork copied every other dbt sidecar table and not this one, so a fork lost its column lineage silently and could not recover it: the cloned digest covers the column edges, so a dynamic run in the fork matched it and stored nothing. - The parquet was collected whole before the edge cap applied, which is exactly the input the cap exists for — a project whose `scan` lineage is quadratic in its widest model could take the worker process down. Decoded a row at a time with the bound enforced during the decode. - The pass swallowed every error from the runner, including the job poller's cancellation and deadline, so a run that blew its timeout inside an optional annotation could still publish a graph and report success. `run_captured` now carries the exit status in its value, so only a failed COMPILE is downgraded, and the pass may spend at most half the remaining wall clock so it cannot starve the build that follows it. - `scan` edges are stored but no longer served: they are most of a project's lineage, nothing renders them, and the graph endpoint is polled by the run page. They are also the first thing the storage cap gives up now, rather than evicting the direct edges the trace draws. - `column_schema` and the column edges take the same gate as the model's SQL. A column-level view is the shape of what the author wrote, one level finer than the `ref()` graph, which is ungated only because it draws relations the caller already sees. - `graph_digest` hashes the new section only when it has edges, so a project that never asked for the pass keeps the digest it has instead of re-snapshotting on every dynamic run until it is redeployed. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01PRtsPQ3Ck69Fu9DNr7bMcJ * fix: the editor buffer's column lineage, and three bounds that were wrong Round-2 review found four defects, all of them introduced by the round-1 fixes. - The `script_visible` gate on the column edges was copied from the node query without its `script_hash IS NULL` arm. `= NULL` is never true, so every version-less row was filtered out and an editor buffer's parse rendered its typed columns and none of their lineage — the one place the feature is meant to be used. Pinned by an assertion in `dbt_pinned_graph.rs`, which is where this class of bug already had a home. - The phase budget was handed to the poller, whose expiry is an `Err` indistinguishable from a cancellation or the job's own deadline, so a slow but valid analysis aborted the build it exists to annotate. The runner gets the full deadline again — those two must still fail the job — and the budget is a race around the whole pass, where expiring is this budget and nothing else. - The decode cap counted parquet ROWS, so `scan` and out-of-graph rows could spend it before a single drawn edge was read. It now counts what is kept, takes direct kinds in a first pass, and is handed the graph's own nodes so the budget cannot go on rows that could never be stored. - Hashing the new digest section conditionally did not preserve old digests, because an absent `column_schema` still serialized as `null` inside the nodes. It is skipped when absent instead. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01PRtsPQ3Ck69Fu9DNr7bMcJ * refactor: split the lineage pass by error contract, and read it in one query Round 3's findings were all consequences of round 1 and 2's fixes, clustered in the same two files, so this reshapes those two seams rather than patching again. The worker pass was one function being three things at once — a subprocess runner with job-lifecycle error semantics, a bounded decoder, and a best-effort degrader — which is why each fix to one perturbed another. It is now `compile_index`, which owns the JOB's semantics (only a cancellation or the job's deadline can `Err`; a non-zero exit, the output ceiling and the phase budget are outcomes), and `read_index`, which owns the ARTIFACT's and knows nothing about the job. The budget wraps the compile alone, so a decode can no longer outlive the timeout that reported the build would get the rest. The output ceiling likewise becomes a value rather than a job error, for the caller that can carry on without the tail of a compile's stdout. The column edges were read by a fourth hand-written copy of the `live`/`chosen` CTEs and the version/editor-buffer join conditions, and copying them is what dropped the `script_hash IS NULL` arm and hid every buffer parse's lineage. Both kinds of edge now come from ONE statement over a `UNION ALL`'d edge source, so those conditions exist once. The union is at the source rather than a join because column lineage can name a node pair `dbt_edge` has no row for: a model reading `{{ this }}` gets edges from itself to itself, and `parent_map` has no self-loop. The cap on the column half now sits after the scope filter, the visibility check and the graph joins — the scope moved into SQL via the existing `ScopePathFilter` — so a row the caller may not read can no longer spend it and leave an allowed project's trace short. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01PRtsPQ3Ck69Fu9DNr7bMcJ * refactor: serve dbt column lineage from its own endpoint The column edges rode on the folder-wide asset graph, which a run page polls, while the trace is drawn for one selected relation. That needed a cap, and a cap has to be applied after every filter that can drop a row. Keyed to the asset there is no cap: `assets/column_lineage` answers for one relation, and the caller's `scripts:read` scope and the project's visibility are decided once, for the script that owns it. Pinning to a run's snapshot or the editor's parse of its buffer costs the job-read gate, so that form is `jobs/dbt_column_lineage/{id}` — the same shape `jobs/dbt_graph/{id}` has. The worker's decode now bounds work and memory separately, and a compile stopped by the output ceiling reports as truncated rather than complete. Co-Authored-By: Claude Opus 5 (1M context) * fix: resolve the owning dbt version the way the graph does The unpinned arm picked the newest live version at the path without narrowing to dbt, so a path since redeployed in another language answered with no lineage while the graph beside it still drew that project's stale nodes. Co-Authored-By: Claude Opus 5 (1M context) * fix: gate pinned column lineage on reading the project, and answer the component Four things round 5 found, three of them in code this branch rewrote: - The pinned arm resolved the version from the job and stopped there, so a share-link viewer entitled to a run got the project's column names and edges while the graph beside it still redacted `raw_code` and `column_schema`. Resolving WHICH version answers is not deciding whether the caller may read it; the version-less editor buffer keeps its exemption, having no `script` row to ask. - The answer was the whole owning project's edges. The canvas lays out the connected component of the selected relation's columns, so the rest was unrenderable weight; a recursive walk over both directions returns exactly what is drawn, and the project key travels with it so a `unique_id` two projects share cannot walk from one graph into the other. - The decode had no exit but the 4M-row backstop once its buckets were full, spending wall clock the build below does not get. - An unreadable index was reported as a missing one, sending the reader to look at their engine rather than at the file. Co-Authored-By: Claude Opus 5 (1M context) * fix: stitch the two column graphs, and walk the component in Rust Round 6's two findings, both regressions this branch introduced: - The decode returned `Continue` on the edge that FILLED the direct-edge budget, so a `scan`-only tail after it decoded to the 4M-row backstop with nowhere to put anything. The read now ends on that edge. - Seam 3 made the pipeline page choose between the dbt graph and the producer one. They share node ids — `// column total <- dbt://wh/analytics/orders.amount` mints the same `(dbt, path, column)` node dbt's own lineage does — so choosing ended a trace at the boundary in both directions. They are merged again, and a ducklake selection asks about the dbt relation its producers name so the chain continues past it. The dbt editor gets the same merge. Also: the component is walked in Rust rather than by a recursive CTE. A CTE has no index, so the recursive term rescanned the doubled edge set once per level — 1243ms against 59ms for the query alone on a 3000-model project, 11.7M rows in the plan. Same answers, same tests; end to end 1.48s to 0.73s there and 1.60s to 0.26s on a 1000-deep chain. The client stops re-asking for a component it already holds, which is most clicks within one project. The four doc sites that described a whole-project answer are rewritten around what it now is, rather than edited where they disagreed. Co-Authored-By: Claude Opus 5 (1M context) * fix: expand every dbt boundary a selection reaches, and only skip what was asked Round 7's findings, all in the frontend seam this branch added: - A ducklake selection seeded the dbt fetch from the FIRST boundary relation it found, so a table derived from two unconnected dbt relations expanded one and left the other a leaf — the same "stops at the boundary" symptom the round-6 fix removed, one hop further along. Every distinct boundary is fetched now and the components merged. - The component cache skipped a relation merely PRESENT in the graph in hand. A relation two projects describe has an owner row in each, and a component fetched for one carries it as an endpoint without the other's half, so that skipped the request that would have resolved the second owner. Only a relation actually asked about under this pin is skipped. - A comment still called the producer graph gated to ducklake selections after it was widened to dbt. Co-Authored-By: Claude Opus 5 (1M context) * refactor: land dbt column lineage as storage and ingest only The API surface that draws a column trace moves to a follow-up PR, on `dbt-column-lineage-surface`. It kept generating findings — a client cache whose premise was wrong for a two-owner relation, then staleness and a lost retry from tightening it, and a seed walk that stopped at the first boundary — and the fix for the last of them is a transitive owner expansion, which has to re-apply the caller's gate to every newly discovered project. That is the same shape as the leak four reviewers caught in the pinned arm, and it wants its own review rather than being the fourth fix at the end of this one. What lands here stands on its own: the analysis pass, `dbt_column_edge`, `dbt_node.column_schema`, the engine gating and the error-contract split — plus the one user-visible half, the typed and ordered column list, which rides the asset graph the details pane already fetches and replaces a panel that could only show the columns an author had documented. Also fixes a real bug in the pass, found in review: it compiled without the build's `--full-refresh`. `is_incremental()` branches on that flag, so an incremental model reading `{{ this }}` compiles its self-join — and any `ref()` inside that branch — only when the flag is absent, and the pass was storing lineage for SQL a full-refresh run never executed. The flag now comes from one place shared with the build, and a run that overrides it gets its own graph rather than standing as the version's. Co-Authored-By: Claude Opus 5 (1M context) * docs: say why direct kinds get the budget without naming a view The bucketing comments explained the priority by what a trace draws, which is a forward reference now that the surface moved out. The reason stands on its own: `copy`/`mod` say the value travelled, `scan` says the column was read to produce the row and so reaches every output column of its model. Co-Authored-By: Claude Opus 5 (1M context) * fix: round-9 findings on the descoped PR - The `full_refresh` helper was inserted between `selection_is_overridden` and its doc comment, so thirteen lines about `select`/`exclude` echoes documented the wrong function and the one they were written for had none. Moved below it. - The parse path ran the analysis compile and the parquet decode BEFORE the guard that returns when there is no warehouse identity, paying for both and dropping the result. Moved after it. - Three sites still described a `/column_lineage` endpoint this branch no longer has, and two user-facing strings promised a column trace it no longer renders: the panel's hint and the descriptor template now say what the flag actually buys, which is the typed column schema. - Dropped test scaffolding the removed suite left behind: a `raw_orders` node and `dbt_edge` whose only assertion re-tested pre-existing graph behaviour, and a second editor-buffer node nothing asserts on. Documented rather than fixed: an incremental model has two shapes, and which one the index holds depends on whether the target existed when the pass ran. `is_incremental()` is false with no target as well as under `--full-refresh`, and dbt has no mode that emits both — so a version's graph describes the compile that produced it, and only a re-ingesting run describes its own run. Co-Authored-By: Claude Opus 5 (1M context) * fix: keep lineage_kind in the edge key, and one answer for --full-refresh - Both unique indexes omitted `lineage_kind`, so a column that is projected AND used as a predicate for the same output column — an ordinary shape — had its `copy` and `scan` edges collapse under `ON CONFLICT DO NOTHING`, while the digest counted both. The kind is part of the fact, so it is part of the key. Edited in the migration rather than added as a second one: it has not landed. - `full_refresh` was shared between the build and the analysis pass without the `command != "test"` condition that sat at the build's call site, so the two disagreed for exactly the runs that build nothing. The condition moved inside the function, which is the point of sharing it, and the command is threaded to the pass. - The "what a trace draws" rewrite missed the copy in `dbt_manifest.rs`. Co-Authored-By: Claude Opus 5 (1M context) * fix: drop the unreachable full_refresh threading, test the uniqueness key `DBT_COMMANDS` is `["build", "retry", "show", "parse"]` and `default_command` returns `build` in every arm, so `command == "test"` cannot happen — the guard the last commit moved into `full_refresh` was already inert where it came from. Threading the command through five signatures to preserve it bought nothing, so it is gone; the build and the pass call one function of the descriptor and the invocation, which is what the sharing was for. The uniqueness-key fix now has a test: a column projected AND used as a predicate for the same output column stores both its `copy` and its `scan` row. Verified against the old key, where it returns 1 instead of 2. Co-Authored-By: Claude Opus 5 (1M context) * fix: restore the dbt test --full-refresh guard I removed on a wrong premise The previous commit removed it after reading `DBT_COMMANDS` and concluding `"test"` was unreachable. That is only true of the command a CALLER can name: `run_dbt` is invoked with `"test"` directly for the `after_all` test phase, so an `after_all` project with `full_refresh: true` reached it — and dbt rejects `--full-refresh` on `test`, failing the phase. Both reviewers caught it. The guard is back inside the shared function, where the build and the pass get one answer, and its doc now records why reading the allowlist alone is misleading. The test covering the `test` case is restored with it. Co-Authored-By: Claude Opus 5 (1M context) * fix: notice a job that ended during the decode, and name truncation as the cause - The parquet decode runs on a blocking thread with no poller watching it, so a cancellation or an expired deadline during it was invisible: `dbt_dep` went on to publish the graph and the job returned success. The job's state is checked once the decode returns, before the caller publishes anything, and an ended job `Err`s — which this module may always do for the job's own semantics. - A compile stopped by the output ceiling could leave no artifact, and the log then blamed the engine's capability, sending the reader to check their adapter rather than the ceiling. Truncation now names itself in the missing and unreadable branches too. Co-Authored-By: Claude Opus 5 (1M context) * fix: read cancellation from the DB after the decode, not from a poller's field `ctx.canceled_by` is only ever written by a poller, and no poller runs during the blocking decode — which is the exact window the check was added for. So the guard caught only a cancellation already observed before it, and the comment beside it claimed more than it did. It now queries `v2_job_queue` directly, the same probe `worker_lockfiles` uses before it overwrites a flow. A failed probe answers "still running": this decides whether to discard work already done, so an unreachable database must not be the reason a healthy deploy loses its graph. Co-Authored-By: Claude Opus 5 (1M context) * refactor: reuse job_is_canceled rather than a second copy of it The probe added last round was `job_is_canceled` from the same file, retyped — same query, same `Connection::Http` behaviour. Reused instead. Its doc said a non-database connection was "a failed probe", which reads as an error path. It is not: it is the agent worker, and on one there is no database to ask, so only the deadline answers and a cancel issued during the decode is not observable. The retry path avoids that by refusing to run on an agent worker at all — which an optional annotation has no business doing — so the gap is recorded at both ends instead. Co-Authored-By: Claude Opus 5 (1M context) * fix: close the agent-worker cancellation gap instead of documenting it The previous commit said a cancel issued during the decode is not observable on an agent worker. It is: `ping_job_status` returns `canceled_by` over both connection kinds, and is how the poller itself notices one there. So the check asks through the ping rather than querying `v2_job_queue` directly, and holds on an agent worker, where a direct query reaches no database at all. `job_is_canceled` goes back to private and its doc to what it said before — the retry that calls it still refuses to run on an agent worker for its own reasons. Co-Authored-By: Claude Opus 5 (1M context) * fix: decode the index under the job poller instead of checking after it Two findings with one cause: the decode was the only phase of this pass with no subprocess behind it, so nothing heartbeated while it ran. A large index left the worker silent for as long as it took, which the zombie sweep reads as a dead job and restarts — and the cancellation check bolted on afterwards could only ever report what had already happened, while dropping the ping's `already_completed`, so a force-cancelled deploy still published its graph. Running it under `run_future_with_polling_update_job_poller` answers all of it: the poller pings throughout, and ends the phase with an `Err` on cancellation, `AlreadyCompleted` or the phase timeout. The bespoke probe is gone with it. Verified on a live deploy: 32 edges and 4 typed schemas ingested through the polled decode. Co-Authored-By: Claude Opus 5 (1M context) * fix: stop a cancelled decode, and say what the read phase can now do Putting the decode under the poller heartbeats it and ends the phase when the job does, but dropping a `JoinHandle` detaches a blocking task rather than cancelling it — so a cancelled job left a thread decoding up to four million rows for a job that was over. The row loop reads an abandonment flag that a drop guard on the awaiting future sets, so the decode stops at its next row. That same change made the read phase able to `Err`, and three places still said it could not — decision 14 in as many words. The distinction that holds is narrower: nothing the ARTIFACT does or fails to do can fail a job, so absent, unreadable and partial are all values; the JOB can still end the phase the read runs in. Stated that way in the module doc, the `Artifact` doc, `MAX_INDEX_ROWS` and the decision. Verified on a live deploy: 32 edges and 4 typed schemas. Co-Authored-By: Claude Opus 5 (1M context) * refactor: share AbortOnDrop, and stop citing a hazard that is now handled `Abandon` was `ansible_executor`'s `AbortOnDrop` retyped — same struct, same reason, same `spawn_blocking` shape. Moved to `common` and used from both. The paragraph explaining why the phase budget wraps the compile alone gave as its reason "a decode still running on a blocking thread", which is exactly what the abandonment flag now prevents. The reason that survives is the one that was always the point: the budget exists to leave the build its share of the clock, and only the compile can spend that share unboundedly. The decode's end is the job's, through the poller it runs under. Co-Authored-By: Claude Opus 5 (1M context) * fix: put both doc comments back on the items they describe Moving AbortOnDrop orphaned a doc at each end: it landed between `raw_to_string`'s doc and `raw_to_string`, and the doc of the struct it replaced stayed behind to prefix `fetch_repo_archive`. Co-Authored-By: Claude Opus 5 (1M context) * docs: name the binding the row loop actually reads `Abandoned` was neither the type nor the binding; the flag is `abandoned`. Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Opus 5 (1M context) --- ...a58ec742d7079703d417c886c8cf0a0e767a6.json | 16 + ...2fa6f3114a151b66b0a116b1d98dd84115188.json | 15 + ...f26b467185edcfde151f22c6e903d15722ad4.json | 15 + ...86f320e4ff58aac8794248925c084707f9b5.json} | 16 +- ...2f8f0ca36920ab66e8922815157a0c147bc47.json | 15 - ...9466279aad62f33c57f392835e8265ed40f26.json | 23 + ...f778b2d3cf8b0f13cf4620192604cc3824a77.json | 14 + ...cad62b5c1eff191f06905a3a78eb600e3bb4b.json | 17 + backend/Cargo.lock | 1 + backend/Cargo.toml | 6 + ...20260904143633_dbt_column_lineage.down.sql | 2 + .../20260904143633_dbt_column_lineage.up.sql | 68 ++ .../parsers/windmill-parser-yaml/src/dbt.rs | 25 + backend/summarized_schema.txt | 4 +- backend/windmill-api-assets/src/lib.rs | 22 +- .../windmill-api-workspaces/src/workspaces.rs | 24 +- backend/windmill-api/openapi.yaml | 18 +- backend/windmill-common/src/dbt_manifest.rs | 270 +++++++- .../tests/dbt_graph_storage.rs | 61 +- backend/windmill-worker/Cargo.toml | 4 + .../windmill-worker/src/ansible_executor.rs | 12 +- backend/windmill-worker/src/common.rs | 14 + .../windmill-worker/src/dbt_column_index.rs | 589 ++++++++++++++++++ backend/windmill-worker/src/dbt_executor.rs | 308 ++++++++- backend/windmill-worker/src/lib.rs | 1 + docs/dbt-runtime.md | 122 +++- .../lib/components/assets/AssetGraph/types.ts | 9 +- .../lib/components/dbt/DbtModelDetails.svelte | 58 +- frontend/src/lib/script_helpers.ts | 6 + 29 files changed, 1660 insertions(+), 95 deletions(-) create mode 100644 backend/.sqlx/query-178cb9cd8dfda66e878cd924f64a58ec742d7079703d417c886c8cf0a0e767a6.json create mode 100644 backend/.sqlx/query-2e562fb35cfa71702e5fdbc505d2fa6f3114a151b66b0a116b1d98dd84115188.json create mode 100644 backend/.sqlx/query-3b7858e47e4e3f31f861e114c7ff26b467185edcfde151f22c6e903d15722ad4.json rename backend/.sqlx/{query-9309262d8a37827e5ab0b3534d1595cb6d94118432fd1770f1f89f7cb52a4f12.json => query-4efca4ff8de0bd297de9eaf0fb7f86f320e4ff58aac8794248925c084707f9b5.json} (70%) delete mode 100644 backend/.sqlx/query-94c0aae349172b9295b81b9e61e2f8f0ca36920ab66e8922815157a0c147bc47.json create mode 100644 backend/.sqlx/query-97fd4f9939d96176f4cfc4a07e69466279aad62f33c57f392835e8265ed40f26.json create mode 100644 backend/.sqlx/query-e5417b36319dafd5fcb2e42d812f778b2d3cf8b0f13cf4620192604cc3824a77.json create mode 100644 backend/.sqlx/query-f49cd9b5ea1e94d7cd10dcd1706cad62b5c1eff191f06905a3a78eb600e3bb4b.json create mode 100644 backend/migrations/20260904143633_dbt_column_lineage.down.sql create mode 100644 backend/migrations/20260904143633_dbt_column_lineage.up.sql create mode 100644 backend/windmill-worker/src/dbt_column_index.rs diff --git a/backend/.sqlx/query-178cb9cd8dfda66e878cd924f64a58ec742d7079703d417c886c8cf0a0e767a6.json b/backend/.sqlx/query-178cb9cd8dfda66e878cd924f64a58ec742d7079703d417c886c8cf0a0e767a6.json new file mode 100644 index 0000000000..7bb2cac5db --- /dev/null +++ b/backend/.sqlx/query-178cb9cd8dfda66e878cd924f64a58ec742d7079703d417c886c8cf0a0e767a6.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM dbt_column_edge\n WHERE workspace_id = $1 AND script_path = $2 AND script_hash = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "178cb9cd8dfda66e878cd924f64a58ec742d7079703d417c886c8cf0a0e767a6" +} diff --git a/backend/.sqlx/query-2e562fb35cfa71702e5fdbc505d2fa6f3114a151b66b0a116b1d98dd84115188.json b/backend/.sqlx/query-2e562fb35cfa71702e5fdbc505d2fa6f3114a151b66b0a116b1d98dd84115188.json new file mode 100644 index 0000000000..7148252002 --- /dev/null +++ b/backend/.sqlx/query-2e562fb35cfa71702e5fdbc505d2fa6f3114a151b66b0a116b1d98dd84115188.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id,\n resource_type, name, asset_path, materialized, materialize_strategy, unique_key,\n tags, description, test_kind, test_column, test_args, severity, attached_node,\n columns, column_schema, freshness, raw_code, original_file_path, ingested_at)\n SELECT $2, script_path, script_hash, job_id, unique_id,\n resource_type, name, asset_path, materialized, materialize_strategy, unique_key,\n tags, description, test_kind, test_column, test_args, severity, attached_node,\n columns, column_schema, freshness, raw_code, original_file_path, ingested_at\n FROM dbt_node\n WHERE workspace_id = $1 AND job_id = '00000000-0000-0000-0000-000000000000'", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "2e562fb35cfa71702e5fdbc505d2fa6f3114a151b66b0a116b1d98dd84115188" +} diff --git a/backend/.sqlx/query-3b7858e47e4e3f31f861e114c7ff26b467185edcfde151f22c6e903d15722ad4.json b/backend/.sqlx/query-3b7858e47e4e3f31f861e114c7ff26b467185edcfde151f22c6e903d15722ad4.json new file mode 100644 index 0000000000..28b34afa97 --- /dev/null +++ b/backend/.sqlx/query-3b7858e47e4e3f31f861e114c7ff26b467185edcfde151f22c6e903d15722ad4.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id,\n parent_unique_id, parent_column, child_unique_id, child_column, lineage_kind,\n ingested_at)\n SELECT $2, script_path, script_hash, job_id, parent_unique_id, parent_column,\n child_unique_id, child_column, lineage_kind, ingested_at\n FROM dbt_column_edge\n WHERE workspace_id = $1 AND job_id = '00000000-0000-0000-0000-000000000000'", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "3b7858e47e4e3f31f861e114c7ff26b467185edcfde151f22c6e903d15722ad4" +} diff --git a/backend/.sqlx/query-9309262d8a37827e5ab0b3534d1595cb6d94118432fd1770f1f89f7cb52a4f12.json b/backend/.sqlx/query-4efca4ff8de0bd297de9eaf0fb7f86f320e4ff58aac8794248925c084707f9b5.json similarity index 70% rename from backend/.sqlx/query-9309262d8a37827e5ab0b3534d1595cb6d94118432fd1770f1f89f7cb52a4f12.json rename to backend/.sqlx/query-4efca4ff8de0bd297de9eaf0fb7f86f320e4ff58aac8794248925c084707f9b5.json index da77dc1de5..134e58ed58 100644 --- a/backend/.sqlx/query-9309262d8a37827e5ab0b3534d1595cb6d94118432fd1770f1f89f7cb52a4f12.json +++ b/backend/.sqlx/query-4efca4ff8de0bd297de9eaf0fb7f86f320e4ff58aac8794248925c084707f9b5.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "WITH live AS (\n -- The graph is stored per deployed VERSION, so this endpoint — which\n -- describes the project as it is now — takes the newest live one per\n -- path. Resolved once here rather than per row: a correlated lookup\n -- on every node is what makes these queries fall over.\n SELECT * FROM (\n SELECT DISTINCT ON (s.path) s.path, s.hash\n FROM script s\n WHERE $5::text IS NULL AND s.workspace_id = $1 AND s.language = 'dbt'\n AND ($3::bigint IS NULL OR s.hash = $3)\n -- A pinned version may be archived by now; that is precisely\n -- the case a historical run needs, so the liveness filter\n -- applies only when picking the current one.\n AND ($3::bigint IS NOT NULL OR (s.deleted = false AND s.archived = false))\n ORDER BY s.path, s.created_at DESC\n ) cur\n UNION ALL\n -- A pinned run names its own version, so `script` is not consulted:\n -- under RLS it would answer for the CALLER's grants on the project,\n -- emptying the graph for a share-link viewer who is entitled to the\n -- run but not the script. A NULL hash here is a job that names no\n -- version at all — an editor buffer parse — and matches only the\n -- version-less rows that parse stored.\n SELECT $5::text, $3::bigint WHERE $5::text IS NOT NULL\n ),\n -- The run's own snapshot when it left one, the version's graph\n -- otherwise. A static descriptor never snapshots, so all of its runs\n -- fall through to the same rows. Existence comes from the marker, not\n -- from a node row: a dynamic run that disabled every model has a\n -- snapshot whose graph is legitimately empty.\n chosen AS (\n -- No visibility check on the job here: reaching this with a job at\n -- all means the caller passed `require_job_read_access` for it, and\n -- re-deciding it under plain RLS can only DISAGREE with that answer\n -- — silently, by falling back to the deployed graph rather than\n -- erroring. A share-link viewer is entitled to the run and would be\n -- shown a different run's model set. See `asset_graph_for`.\n SELECT CASE WHEN $4::uuid IS NOT NULL AND EXISTS (\n SELECT 1 FROM dbt_graph_snapshot g\n WHERE g.workspace_id = $1 AND g.job_id = $4)\n THEN $4::uuid\n ELSE '00000000-0000-0000-0000-000000000000'::uuid END AS job_id\n ),\n scoped AS (\n SELECT n.script_path, n.unique_id FROM dbt_node n\n -- `=` still, with the NULL-to-NULL case spelled out and gated on\n -- the pin: a version-less row's hash is NULL on both sides, which\n -- `=` never matches, but `IS NOT DISTINCT FROM` would cost the\n -- equality its index bound on the UNPINNED workspace graph — the\n -- hot path. Unpinned, `$5` is NULL and the second arm folds away.\n JOIN live l ON l.path = n.script_path\n AND (n.script_hash = l.hash\n OR ($5::text IS NOT NULL AND l.hash IS NULL\n AND n.script_hash IS NULL))\n JOIN chosen ch ON ch.job_id = n.job_id\n WHERE n.workspace_id = $1 AND n.asset_path IS NOT NULL\n -- Unpinned, the scope is the relations in view: `asset` says\n -- which of them this folder touches. Pinned, that table is the\n -- WRONG scope — it holds one row set per path, describing the\n -- current deploy, so a model this version had and the current one\n -- dropped would be filtered out of its own run's graph. The\n -- pinned graph's nodes are the scope. Keyed on the pin rather\n -- than on the hash: an editor parse pins without naming one, and\n -- its models are precisely the ones `asset` does not know yet.\n AND ($3::bigint IS NOT NULL OR $5::text IS NOT NULL\n OR n.asset_path IN (\n SELECT path FROM asset\n WHERE workspace_id = $1 AND kind = 'dbt'\n AND ($2::text IS NULL OR usage_path LIKE $2)))\n )\n SELECT n.script_path AS \"script_path!\", n.unique_id AS \"unique_id!\",\n n.resource_type AS \"resource_type!\", n.name AS \"name!\", n.asset_path,\n n.materialized, n.materialize_strategy, n.tags AS \"tags!\", n.description,\n n.test_kind, n.test_column, n.test_args, n.severity, n.attached_node,\n n.columns, n.freshness,\n n.raw_code, n.original_file_path,\n -- Whether the caller may read the project this row describes.\n -- The query deliberately reaches outside the requested folder\n -- so an in-scope consumer can explain the relation it reads,\n -- and `dbt_node` carries no RLS of its own; the relation's\n -- SHAPE is fine to answer that way, everything the project's\n -- author WROTE is not. Applied in Rust, over one predicate, so\n -- the fields it covers are named in one place. This runs in the\n -- authed transaction, so `script`'s RLS answers it. Matched on\n -- the HASH as well: `extra_perms` is per row, so a path\n -- recreated with narrower ones leaves the archived version\n -- readable, and a path-only probe would answer for THAT grant\n -- while returning this version's source.\n --\n -- A version-less row has no `script` row to ask, and needs\n -- none: it exists only because this caller's own parse job\n -- created it from a buffer they wrote, and the unpinned `live`\n -- branch — fed from `script` — can never join to one.\n (n.script_hash IS NULL OR EXISTS (\n SELECT 1 FROM script sc\n WHERE sc.workspace_id = n.workspace_id AND sc.path = n.script_path\n AND sc.hash = n.script_hash\n )) AS \"script_visible!\"\n FROM dbt_node n\n JOIN live l ON l.path = n.script_path\n AND (n.script_hash = l.hash\n OR ($5::text IS NOT NULL AND l.hash IS NULL\n AND n.script_hash IS NULL))\n -- Every join onto `dbt_node` needs this, not just the scoping CTE:\n -- `job_id` is part of the key, so without it each model comes back\n -- once per retained snapshot plus once for the version's graph.\n JOIN chosen ch ON ch.job_id = n.job_id\n WHERE n.workspace_id = $1\n -- Joined on BOTH columns: a dbt `unique_id` is project-local, so\n -- two projects with the same model name would otherwise pull each\n -- other's rows.\n AND (EXISTS (SELECT 1 FROM scoped s\n WHERE s.script_path = n.script_path\n AND s.unique_id = n.unique_id)\n OR EXISTS (SELECT 1 FROM scoped s\n WHERE s.script_path = n.script_path\n AND s.unique_id = n.attached_node))\n ORDER BY n.script_path, n.unique_id", + "query": "WITH live AS (\n -- The graph is stored per deployed VERSION, so this endpoint — which\n -- describes the project as it is now — takes the newest live one per\n -- path. Resolved once here rather than per row: a correlated lookup\n -- on every node is what makes these queries fall over.\n SELECT * FROM (\n SELECT DISTINCT ON (s.path) s.path, s.hash\n FROM script s\n WHERE $5::text IS NULL AND s.workspace_id = $1 AND s.language = 'dbt'\n AND ($3::bigint IS NULL OR s.hash = $3)\n -- A pinned version may be archived by now; that is precisely\n -- the case a historical run needs, so the liveness filter\n -- applies only when picking the current one.\n AND ($3::bigint IS NOT NULL OR (s.deleted = false AND s.archived = false))\n ORDER BY s.path, s.created_at DESC\n ) cur\n UNION ALL\n -- A pinned run names its own version, so `script` is not consulted:\n -- under RLS it would answer for the CALLER's grants on the project,\n -- emptying the graph for a share-link viewer who is entitled to the\n -- run but not the script. A NULL hash here is a job that names no\n -- version at all — an editor buffer parse — and matches only the\n -- version-less rows that parse stored.\n SELECT $5::text, $3::bigint WHERE $5::text IS NOT NULL\n ),\n -- The run's own snapshot when it left one, the version's graph\n -- otherwise. A static descriptor never snapshots, so all of its runs\n -- fall through to the same rows. Existence comes from the marker, not\n -- from a node row: a dynamic run that disabled every model has a\n -- snapshot whose graph is legitimately empty.\n chosen AS (\n -- No visibility check on the job here: reaching this with a job at\n -- all means the caller passed `require_job_read_access` for it, and\n -- re-deciding it under plain RLS can only DISAGREE with that answer\n -- — silently, by falling back to the deployed graph rather than\n -- erroring. A share-link viewer is entitled to the run and would be\n -- shown a different run's model set. See `asset_graph_for`.\n SELECT CASE WHEN $4::uuid IS NOT NULL AND EXISTS (\n SELECT 1 FROM dbt_graph_snapshot g\n WHERE g.workspace_id = $1 AND g.job_id = $4)\n THEN $4::uuid\n ELSE '00000000-0000-0000-0000-000000000000'::uuid END AS job_id\n ),\n scoped AS (\n SELECT n.script_path, n.unique_id FROM dbt_node n\n -- `=` still, with the NULL-to-NULL case spelled out and gated on\n -- the pin: a version-less row's hash is NULL on both sides, which\n -- `=` never matches, but `IS NOT DISTINCT FROM` would cost the\n -- equality its index bound on the UNPINNED workspace graph — the\n -- hot path. Unpinned, `$5` is NULL and the second arm folds away.\n JOIN live l ON l.path = n.script_path\n AND (n.script_hash = l.hash\n OR ($5::text IS NOT NULL AND l.hash IS NULL\n AND n.script_hash IS NULL))\n JOIN chosen ch ON ch.job_id = n.job_id\n WHERE n.workspace_id = $1 AND n.asset_path IS NOT NULL\n -- Unpinned, the scope is the relations in view: `asset` says\n -- which of them this folder touches. Pinned, that table is the\n -- WRONG scope — it holds one row set per path, describing the\n -- current deploy, so a model this version had and the current one\n -- dropped would be filtered out of its own run's graph. The\n -- pinned graph's nodes are the scope. Keyed on the pin rather\n -- than on the hash: an editor parse pins without naming one, and\n -- its models are precisely the ones `asset` does not know yet.\n AND ($3::bigint IS NOT NULL OR $5::text IS NOT NULL\n OR n.asset_path IN (\n SELECT path FROM asset\n WHERE workspace_id = $1 AND kind = 'dbt'\n AND ($2::text IS NULL OR usage_path LIKE $2)))\n )\n SELECT n.script_path AS \"script_path!\", n.unique_id AS \"unique_id!\",\n n.resource_type AS \"resource_type!\", n.name AS \"name!\", n.asset_path,\n n.materialized, n.materialize_strategy, n.tags AS \"tags!\", n.description,\n n.test_kind, n.test_column, n.test_args, n.severity, n.attached_node,\n n.columns, n.column_schema, n.freshness,\n n.raw_code, n.original_file_path,\n -- Whether the caller may read the project this row describes.\n -- The query deliberately reaches outside the requested folder\n -- so an in-scope consumer can explain the relation it reads,\n -- and `dbt_node` carries no RLS of its own; the relation's\n -- SHAPE is fine to answer that way, everything the project's\n -- author WROTE is not. Applied in Rust, over one predicate, so\n -- the fields it covers are named in one place. This runs in the\n -- authed transaction, so `script`'s RLS answers it. Matched on\n -- the HASH as well: `extra_perms` is per row, so a path\n -- recreated with narrower ones leaves the archived version\n -- readable, and a path-only probe would answer for THAT grant\n -- while returning this version's source.\n --\n -- A version-less row has no `script` row to ask, and needs\n -- none: it exists only because this caller's own parse job\n -- created it from a buffer they wrote, and the unpinned `live`\n -- branch — fed from `script` — can never join to one.\n (n.script_hash IS NULL OR EXISTS (\n SELECT 1 FROM script sc\n WHERE sc.workspace_id = n.workspace_id AND sc.path = n.script_path\n AND sc.hash = n.script_hash\n )) AS \"script_visible!\"\n FROM dbt_node n\n JOIN live l ON l.path = n.script_path\n AND (n.script_hash = l.hash\n OR ($5::text IS NOT NULL AND l.hash IS NULL\n AND n.script_hash IS NULL))\n -- Every join onto `dbt_node` needs this, not just the scoping CTE:\n -- `job_id` is part of the key, so without it each model comes back\n -- once per retained snapshot plus once for the version's graph.\n JOIN chosen ch ON ch.job_id = n.job_id\n WHERE n.workspace_id = $1\n -- Joined on BOTH columns: a dbt `unique_id` is project-local, so\n -- two projects with the same model name would otherwise pull each\n -- other's rows.\n AND (EXISTS (SELECT 1 FROM scoped s\n WHERE s.script_path = n.script_path\n AND s.unique_id = n.unique_id)\n OR EXISTS (SELECT 1 FROM scoped s\n WHERE s.script_path = n.script_path\n AND s.unique_id = n.attached_node))\n ORDER BY n.script_path, n.unique_id", "describe": { "columns": [ { @@ -80,21 +80,26 @@ }, { "ordinal": 15, - "name": "freshness", + "name": "column_schema", "type_info": "Jsonb" }, { "ordinal": 16, + "name": "freshness", + "type_info": "Jsonb" + }, + { + "ordinal": 17, "name": "raw_code", "type_info": "Text" }, { - "ordinal": 17, + "ordinal": 18, "name": "original_file_path", "type_info": "Text" }, { - "ordinal": 18, + "ordinal": 19, "name": "script_visible!", "type_info": "Bool" } @@ -127,8 +132,9 @@ true, true, true, + true, null ] }, - "hash": "9309262d8a37827e5ab0b3534d1595cb6d94118432fd1770f1f89f7cb52a4f12" + "hash": "4efca4ff8de0bd297de9eaf0fb7f86f320e4ff58aac8794248925c084707f9b5" } diff --git a/backend/.sqlx/query-94c0aae349172b9295b81b9e61e2f8f0ca36920ab66e8922815157a0c147bc47.json b/backend/.sqlx/query-94c0aae349172b9295b81b9e61e2f8f0ca36920ab66e8922815157a0c147bc47.json deleted file mode 100644 index 823a65cda3..0000000000 --- a/backend/.sqlx/query-94c0aae349172b9295b81b9e61e2f8f0ca36920ab66e8922815157a0c147bc47.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id,\n resource_type, name, asset_path, materialized, materialize_strategy, unique_key,\n tags, description, test_kind, test_column, test_args, severity, attached_node,\n columns, freshness, raw_code, original_file_path, ingested_at)\n SELECT $2, script_path, script_hash, job_id, unique_id,\n resource_type, name, asset_path, materialized, materialize_strategy, unique_key,\n tags, description, test_kind, test_column, test_args, severity, attached_node,\n columns, freshness, raw_code, original_file_path, ingested_at\n FROM dbt_node\n WHERE workspace_id = $1 AND job_id = '00000000-0000-0000-0000-000000000000'", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Varchar" - ] - }, - "nullable": [] - }, - "hash": "94c0aae349172b9295b81b9e61e2f8f0ca36920ab66e8922815157a0c147bc47" -} diff --git a/backend/.sqlx/query-97fd4f9939d96176f4cfc4a07e69466279aad62f33c57f392835e8265ed40f26.json b/backend/.sqlx/query-97fd4f9939d96176f4cfc4a07e69466279aad62f33c57f392835e8265ed40f26.json new file mode 100644 index 0000000000..b98a71fb8a --- /dev/null +++ b/backend/.sqlx/query-97fd4f9939d96176f4cfc4a07e69466279aad62f33c57f392835e8265ed40f26.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT count(*) FROM dbt_column_edge WHERE workspace_id = $1 AND script_hash = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Int8" + ] + }, + "nullable": [ + null + ] + }, + "hash": "97fd4f9939d96176f4cfc4a07e69466279aad62f33c57f392835e8265ed40f26" +} diff --git a/backend/.sqlx/query-e5417b36319dafd5fcb2e42d812f778b2d3cf8b0f13cf4620192604cc3824a77.json b/backend/.sqlx/query-e5417b36319dafd5fcb2e42d812f778b2d3cf8b0f13cf4620192604cc3824a77.json new file mode 100644 index 0000000000..797fee956b --- /dev/null +++ b/backend/.sqlx/query-e5417b36319dafd5fcb2e42d812f778b2d3cf8b0f13cf4620192604cc3824a77.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM dbt_column_edge\n WHERE job_id <> '00000000-0000-0000-0000-000000000000'\n AND ingested_at < now() - make_interval(days => $1)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4" + ] + }, + "nullable": [] + }, + "hash": "e5417b36319dafd5fcb2e42d812f778b2d3cf8b0f13cf4620192604cc3824a77" +} diff --git a/backend/.sqlx/query-f49cd9b5ea1e94d7cd10dcd1706cad62b5c1eff191f06905a3a78eb600e3bb4b.json b/backend/.sqlx/query-f49cd9b5ea1e94d7cd10dcd1706cad62b5c1eff191f06905a3a78eb600e3bb4b.json new file mode 100644 index 0000000000..75526d2db8 --- /dev/null +++ b/backend/.sqlx/query-f49cd9b5ea1e94d7cd10dcd1706cad62b5c1eff191f06905a3a78eb600e3bb4b.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM dbt_column_edge WHERE workspace_id = $1 AND script_path = $2\n AND script_hash = $3 AND job_id = $4", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "f49cd9b5ea1e94d7cd10dcd1706cad62b5c1eff191f06905a3a78eb600e3bb4b" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 9a458a8713..5a19b73d1c 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -16699,6 +16699,7 @@ dependencies = [ "opentelemetry 0.30.0", "opentelemetry-proto 0.30.0", "oracle", + "parquet", "pem 3.0.6", "pep440_rs", "postgres-native-tls 0.5.3", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index f0bccb6fa8..9ef1d099ee 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -665,6 +665,12 @@ process-wrap = { version = "8.2.1", features = ["tokio1"] } systemstat = "0.2.4" datafusion = "47.0.0" +# The row API only: a dbt engine's parquet index is six string columns, so this +# needs no arrow and no writer. `parquet` is already in the tree with `arrow` for +# every shipped edition (`oss_core`), and cargo unifies the features there; this +# set is what a build WITHOUT object storage compiles. ZSTD is what the engine +# writes today, snap what parquet writers most often default to. +parquet = { version = "55.2.0", default-features = false, features = ["snap", "zstd"] } object_store = { git = "https://github.com/apache/arrow-rs-object-store", rev = "36752c975d4f29e20b57c91f81a10872dcd48ae7", features = ["aws", "azure", "gcp"] } openidconnect = { version = "4.0.0-rc.1" } aws-config = "^1" diff --git a/backend/migrations/20260904143633_dbt_column_lineage.down.sql b/backend/migrations/20260904143633_dbt_column_lineage.down.sql new file mode 100644 index 0000000000..a053a94291 --- /dev/null +++ b/backend/migrations/20260904143633_dbt_column_lineage.down.sql @@ -0,0 +1,2 @@ +DROP TABLE IF EXISTS dbt_column_edge; +ALTER TABLE dbt_node DROP COLUMN IF EXISTS column_schema; diff --git a/backend/migrations/20260904143633_dbt_column_lineage.up.sql b/backend/migrations/20260904143633_dbt_column_lineage.up.sql new file mode 100644 index 0000000000..cb463c249f --- /dev/null +++ b/backend/migrations/20260904143633_dbt_column_lineage.up.sql @@ -0,0 +1,68 @@ +-- Column-level lineage, from the engine's own static analysis. +-- +-- `manifest.json` carries none, which is why decision 14 recorded the feature as +-- unavailable. The edges exist in a different artifact: an engine that does +-- static analysis writes `target/index/dbt.column_lineage.parquet` under +-- `dbt compile --static-analysis strict --write-index`. That pass is opt-in per +-- project (`column_lineage: true`), because strict analysis rejects SQL the +-- default accepts and must never become a silent requirement of running a build. + +-- One column-to-column edge, keyed exactly like `dbt_edge`: a version's graph +-- dies with its version through the composite foreign key, a run's snapshot is +-- keyed by `job_id` with the zero UUID meaning "the version's own graph", and an +-- editor buffer's parse carries a NULL `script_hash` keyed to its preview job. +CREATE TABLE IF NOT EXISTS dbt_column_edge ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE ON UPDATE CASCADE, + script_path VARCHAR(255) NOT NULL, + script_hash BIGINT, + job_id UUID NOT NULL DEFAULT '00000000-0000-0000-0000-000000000000', + parent_unique_id TEXT NOT NULL, + parent_column TEXT NOT NULL, + child_unique_id TEXT NOT NULL, + child_column TEXT NOT NULL, + -- dbt's own word for how the value travelled: `copy` (passthrough), `mod` + -- (transformed), `scan` (the column was read to produce the ROW rather than + -- the value -- a join key, a `where` predicate, a `group by`). TEXT rather + -- than an enum because the engine treats the set as open: its own reader maps + -- those three and returns anything else verbatim. + lineage_kind TEXT NOT NULL, + ingested_at TIMESTAMPTZ NOT NULL DEFAULT now(), + -- Two partial unique indexes rather than a primary key, for the reason + -- 20260801121717 gives: a versioned graph is keyed by its version, a buffer + -- parse by its job alone. `lineage_kind` is part of the key because it is part + -- of the fact: a column that is both projected and used as a predicate for the + -- same output column has a `copy` edge AND a `scan` one, and the digest counts + -- both. Leaving it out let `ON CONFLICT DO NOTHING` drop the second while the + -- digest still claimed it was stored. + CONSTRAINT dbt_column_edge_script_fkey FOREIGN KEY (workspace_id, script_hash) + REFERENCES script (workspace_id, hash) ON DELETE CASCADE ON UPDATE CASCADE +); + +CREATE UNIQUE INDEX IF NOT EXISTS dbt_column_edge_versioned_key + ON dbt_column_edge (workspace_id, script_path, script_hash, job_id, + parent_unique_id, parent_column, child_unique_id, child_column, + lineage_kind) + WHERE script_hash IS NOT NULL; +CREATE UNIQUE INDEX IF NOT EXISTS dbt_column_edge_editor_key + ON dbt_column_edge (workspace_id, job_id, + parent_unique_id, parent_column, child_unique_id, child_column, + lineage_kind) + WHERE script_hash IS NULL; + +-- Same age sweep as the other per-run rows, and the same reason there is no +-- foreign key to `v2_job`. +CREATE INDEX IF NOT EXISTS idx_dbt_column_edge_run_age ON dbt_column_edge (ingested_at) + WHERE job_id <> '00000000-0000-0000-0000-000000000000'; + +-- The real column schema of a node, which only static analysis knows: an +-- ordered `[{"name": …, "type": …}]`, from `dbt.node_columns.parquet`. +-- +-- Beside `columns` rather than folded into it. `columns` is the DECLARED +-- metadata `manifest.json` carries -- the names an author wrote in `schema.yml` +-- and the prose against them -- and stays exactly that, so a project that +-- documents two of forty columns keeps saying so. This is the other forty, +-- typed, in the order the model produces them. +ALTER TABLE dbt_node ADD COLUMN IF NOT EXISTS column_schema JSONB; + +GRANT ALL ON dbt_column_edge TO windmill_user; +GRANT ALL ON dbt_column_edge TO windmill_admin; diff --git a/backend/parsers/windmill-parser-yaml/src/dbt.rs b/backend/parsers/windmill-parser-yaml/src/dbt.rs index 980b547c68..36617c5639 100644 --- a/backend/parsers/windmill-parser-yaml/src/dbt.rs +++ b/backend/parsers/windmill-parser-yaml/src/dbt.rs @@ -59,6 +59,19 @@ impl DbtEngine { matches!(self, DbtEngine::DbtCore1x) } + /// Whether the engine's CLI has `--write-index`, the flag that writes the + /// parquet index column lineage lives in. False for 1.x, whose Python CLI + /// has no such option. + /// + /// True is not a promise that the artifact appears: `dbt-core` 2.0.0-alpha.5 + /// accepts the flag, declares the views over `dbt.column_lineage` in its own + /// `views.sql`, and writes neither that parquet nor `dbt.node_columns`. Only + /// Fusion does today. Attempting the pass on both is what lets a later 2.x + /// release pick the feature up with no change here. + pub fn writes_column_index(&self) -> bool { + !matches!(self, DbtEngine::DbtCore1x) + } + /// Whether the engine has `--defer-state`, the deferral-only half of /// `--state`. /// @@ -139,6 +152,18 @@ pub struct DbtDescriptor { pub selector: Option, #[serde(default)] pub test_behavior: DbtTestBehavior, + /// Ingest column-to-column lineage and the real column schemas, from the + /// engine's static analysis. + /// + /// Opt-in, and it has to be: the artifact only appears under + /// `--static-analysis strict`, which rejects SQL the default accepts (an + /// unresolvable identifier is an error there and compiles fine otherwise). + /// Turning it on for everyone would make a stricter dialect the price of + /// deploying a dbt project. It is a separate `dbt compile` pass, so nothing + /// it decides can change what a build does; a project it cannot analyze + /// keeps the graph it has today. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub column_lineage: bool, /// `--vars`. dbt vars are typed — numbers, booleans, lists and objects are /// all normal — so values keep their YAML type; only string leaves carry /// `{{ arg }}` placeholders the worker substitutes from job args. Coercing diff --git a/backend/summarized_schema.txt b/backend/summarized_schema.txt index 5ab10d1e02..67758cef32 100644 --- a/backend/summarized_schema.txt +++ b/backend/summarized_schema.txt @@ -70,13 +70,15 @@ ci_test_reference: workspace_id(char), test_script_path(char), test_script_hash( concurrency_settings: hash(bigint), concurrency_key(char), concurrent_limit(int), concurrency_time_window_s(int) config: name(char), config(jsonb) custom_concurrency_key_ended: key(char), ended_at(ts) +dbt_column_edge: workspace_id(char), script_path(char), script_hash(bigint), job_id(uuid), parent_unique_id(text), parent_column(text), child_unique_id(text), child_column(text), lineage_kind(text), ingested_at(ts) + FK: (workspace_id) -> workspace(id), (workspace_id, script_hash) -> script(workspace_id, hash) dbt_edge: workspace_id(char), script_path(char), script_hash(bigint), job_id(uuid), parent_unique_id(text), child_unique_id(text), ingested_at(ts) FK: (workspace_id) -> workspace(id), (workspace_id, script_hash) -> script(workspace_id, hash) dbt_graph_snapshot: workspace_id(char), script_path(char), script_hash(bigint), job_id(uuid), digest(text), relation_root_at_last_ingest(text), ingested_at(ts), permissioned_as(char) FK: (workspace_id) -> workspace(id), (workspace_id, script_hash) -> script(workspace_id, hash) dbt_environment_state: workspace_id(char), script_path(char), environment(text), job_id(uuid), manifest(text), manifest_key(text), run_results(text), run_results_key(text), updated_at(ts) FK: (workspace_id) -> workspace(id) -dbt_node: workspace_id(char), script_path(char), script_hash(bigint), job_id(uuid), unique_id(text), resource_type(text), name(text), asset_path(text), materialized(text), materialize_strategy(text), unique_key(text), tags(text[]), description(text), test_kind(text), test_column(text), test_args(jsonb), severity(text), attached_node(text), columns(jsonb), freshness(jsonb), raw_code(text), original_file_path(text), ingested_at(ts) +dbt_node: workspace_id(char), script_path(char), script_hash(bigint), job_id(uuid), unique_id(text), resource_type(text), name(text), asset_path(text), materialized(text), materialize_strategy(text), unique_key(text), tags(text[]), description(text), test_kind(text), test_column(text), test_args(jsonb), severity(text), attached_node(text), columns(jsonb), column_schema(jsonb), freshness(jsonb), raw_code(text), original_file_path(text), ingested_at(ts) FK: (workspace_id) -> workspace(id), (workspace_id, script_hash) -> script(workspace_id, hash) dbt_run_progress: workspace_id(char), job_id(uuid), asset_kind(asset_kind), asset_path(char), status(materialization_status), row_count(bigint), error(text), updated_at(ts) FK: (workspace_id) -> workspace(id) diff --git a/backend/windmill-api-assets/src/lib.rs b/backend/windmill-api-assets/src/lib.rs index 09720b2203..51bfa6c2c7 100644 --- a/backend/windmill-api-assets/src/lib.rs +++ b/backend/windmill-api-assets/src/lib.rs @@ -663,10 +663,21 @@ struct DbtAssetProvenance { description: Option, #[serde(skip_serializing_if = "Vec::is_empty")] data_tests: Vec, - /// Declared column metadata (name -> description). NOT column lineage — - /// `manifest.json` carries none (docs/dbt-runtime.md, decision 14). + /// Declared column metadata (name -> description): what `manifest.json` + /// carries, which is only the columns an author wrote down. #[serde(skip_serializing_if = "Option::is_none")] columns: Option, + /// Every column of the relation, typed and in order — + /// `[{"name": …, "type": …}]` — from the engine's static analysis. Present + /// only for a project that opted into it. + /// + /// Gated exactly like `columns` and the model's SQL: a full column list is + /// the shape of what the author WROTE, one level finer than the `ref()` + /// graph, which is ungated only because it draws relations the caller + /// already sees in `asset`. Widening that boundary has to be a decision, not + /// a consequence of a project turning the analysis pass on. + #[serde(skip_serializing_if = "Option::is_none")] + column_schema: Option, /// A source's declared freshness policy, for the staleness chip. #[serde(skip_serializing_if = "Option::is_none")] freshness: Option, @@ -1323,7 +1334,7 @@ pub async fn asset_graph_for( n.resource_type AS "resource_type!", n.name AS "name!", n.asset_path, n.materialized, n.materialize_strategy, n.tags AS "tags!", n.description, n.test_kind, n.test_column, n.test_args, n.severity, n.attached_node, - n.columns, n.freshness, + n.columns, n.column_schema, n.freshness, n.raw_code, n.original_file_path, -- Whether the caller may read the project this row describes. -- The query deliberately reaches outside the requested folder @@ -1379,6 +1390,10 @@ pub async fn asset_graph_for( // `ref()` lineage between two models, resolved to the relations they // produce. Joined to `dbt_node` on both key columns because a dbt // `unique_id` is only unique within its project. + // + // Column lineage is NOT here. It is stored per relation and per column, and + // this response is folder-wide and polled by a run page, so it carries only + // what the canvas draws for every node at once. let dbt_edge_rows = sqlx::query!( r#"WITH live AS ( SELECT * FROM ( @@ -1604,6 +1619,7 @@ pub async fn asset_graph_for( description: r.description.clone().filter(|_| source_allowed), data_tests: vec![], columns: r.columns.clone().filter(|_| source_allowed), + column_schema: r.column_schema.clone().filter(|_| source_allowed), freshness: r.freshness.clone().filter(|_| source_allowed), }; // One relation can carry rows from several projects — typically a model diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index c9343840d2..93e0ad5ca8 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -6627,7 +6627,7 @@ async fn clone_scripts( } /// The parsed dbt graph a deployed script carries: its models, their SQL and -/// tests, and the `ref()` lineage between them. +/// tests, and the `ref()` and column-level lineage between them. /// /// Keyed on (workspace_id, script_path, script_hash), and the fork keeps every /// script's hash, so each row moves across as itself. @@ -6645,11 +6645,11 @@ async fn clone_dbt_graph( "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id, resource_type, name, asset_path, materialized, materialize_strategy, unique_key, tags, description, test_kind, test_column, test_args, severity, attached_node, - columns, freshness, raw_code, original_file_path, ingested_at) + columns, column_schema, freshness, raw_code, original_file_path, ingested_at) SELECT $2, script_path, script_hash, job_id, unique_id, resource_type, name, asset_path, materialized, materialize_strategy, unique_key, tags, description, test_kind, test_column, test_args, severity, attached_node, - columns, freshness, raw_code, original_file_path, ingested_at + columns, column_schema, freshness, raw_code, original_file_path, ingested_at FROM dbt_node WHERE workspace_id = $1 AND job_id = '00000000-0000-0000-0000-000000000000'", source_workspace_id, @@ -6669,6 +6669,24 @@ async fn clone_dbt_graph( ) .execute(&mut **tx) .await?; + // Column lineage travels with the rest of the graph, and it has to: the + // snapshot's digest covers it, so a fork missing these rows recomputes the + // digest the source stored, matches, and stores nothing — leaving the + // lineage gone until someone redeploys, which is the failure this whole + // function exists to prevent. + sqlx::query!( + "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id, + parent_unique_id, parent_column, child_unique_id, child_column, lineage_kind, + ingested_at) + SELECT $2, script_path, script_hash, job_id, parent_unique_id, parent_column, + child_unique_id, child_column, lineage_kind, ingested_at + FROM dbt_column_edge + WHERE workspace_id = $1 AND job_id = '00000000-0000-0000-0000-000000000000'", + source_workspace_id, + target_workspace_id + ) + .execute(&mut **tx) + .await?; sqlx::query!( "INSERT INTO dbt_graph_snapshot (workspace_id, script_path, script_hash, job_id, digest, relation_root_at_last_ingest, ingested_at) diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 5c0a6d57c6..f6960126b1 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -26276,7 +26276,23 @@ components: columns: type: object additionalProperties: true - description: Declared column metadata (name -> description). NOT column lineage — `manifest.json` carries none. + description: Declared column metadata (name -> description) — what `manifest.json` carries, which is only the columns an author wrote down. Omitted when the caller cannot read the script. + column_schema: + type: array + description: >- + Every column of the relation, typed and in the order the model + produces them, from the engine's static analysis. Present only for a + project that opted into it, and gated like `columns` and the model's + SQL: a full column list is the shape of what the author wrote. + items: + type: object + required: [name] + properties: + name: + type: string + type: + type: string + description: The declared type where `schema.yml` gives one, else the inferred one. Omitted when neither is known. freshness: type: object additionalProperties: true diff --git a/backend/windmill-common/src/dbt_manifest.rs b/backend/windmill-common/src/dbt_manifest.rs index 22ac6ddf33..157a61f129 100644 --- a/backend/windmill-common/src/dbt_manifest.rs +++ b/backend/windmill-common/src/dbt_manifest.rs @@ -190,6 +190,19 @@ fn graph_digest(ingested: &IngestedManifest, relation_root: &str) -> String { .unwrap_or_default() .as_bytes(), ); + // Only when there are any, so a project that never asked for the analysis + // pass keeps the digest it already has. Hashing an empty section + // unconditionally would change every stored digest at once, and every + // dynamic run would then store a full snapshot until its script is + // redeployed — which reads exactly like the suppression above never working. + if !ingested.column_edges.is_empty() { + h.update(b"\0"); + h.update( + serde_json::to_string(&ingested.column_edges) + .unwrap_or_default() + .as_bytes(), + ); + } format!("{:x}", h.finalize()) } @@ -299,6 +312,14 @@ pub async fn prune_dbt_run_graphs( ) .execute(db) .await?; + sqlx::query!( + "DELETE FROM dbt_column_edge + WHERE job_id <> '00000000-0000-0000-0000-000000000000' + AND ingested_at < now() - make_interval(days => $1)", + RUN_GRAPH_RETENTION_DAYS, + ) + .execute(db) + .await?; // In ONE transaction with the orphan sweep: a restart in the gap leaves graph // rows whose marker is gone, and since the sweep runs only when a marker went, // every later call computes `retired == 0` and skips them for good. @@ -327,7 +348,7 @@ pub async fn prune_dbt_run_graphs( // partial index here — all of them `WHERE job_id <> DEPLOYED` — and past the // keep-count is rare, so the ordinary run should pay for neither. if retired > 0 { - for table in ["dbt_node", "dbt_edge"] { + for table in ["dbt_node", "dbt_edge", "dbt_column_edge"] { sqlx::query(&format!( "DELETE FROM {table} t WHERE t.workspace_id = $1 AND t.script_path = $2 @@ -383,6 +404,18 @@ pub struct IngestedNode { pub severity: Option, pub attached_node: Option, pub columns: Option, + /// The node's real columns, typed and ordered — `[{"name": …, "type": …}]`, + /// from the engine's static analysis. `None` when the project did not ask + /// for it or the engine wrote none. Beside `columns` rather than merged into + /// it: that one is what the author DECLARED, and stays that. + /// + /// Skipped when absent, unlike its neighbours, because `graph_digest` + /// serializes these nodes: emitting `"column_schema":null` would change + /// every stored digest at once, and every dynamic run of a project that + /// never asked for the pass would store a full snapshot until its script is + /// redeployed. + #[serde(skip_serializing_if = "Option::is_none")] + pub column_schema: Option, pub freshness: Option, /// The transform itself, for the graph to render. The copy taken at /// deploy: the file itself is in the script's module bundle. @@ -390,6 +423,40 @@ pub struct IngestedNode { pub original_file_path: Option, } +/// One column-to-column edge of the ingested graph. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Default)] +#[serde(default)] +pub struct IngestedColumnEdge { + pub parent_unique_id: String, + pub parent_column: String, + pub child_unique_id: String, + pub child_column: String, + /// dbt's own word: `copy`, `mod` or `scan`. Kept verbatim — the engine's own + /// reader maps those three and passes anything else through, so the set is + /// open. + pub lineage_kind: String, +} + +/// One column of a node, as the engine's static analysis resolved it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct IndexedColumn { + pub name: String, + /// The declared type where `schema.yml` gives one, else the inferred one. + /// Empty when neither is known; the column still belongs to the relation, so + /// only the type is left out. + pub column_type: String, + /// Position in the relation, which is the order the panel lists them in. + pub index: i64, +} + +/// What one `--write-index` pass produced: the column edges of the whole +/// project and the real column schema per node. +#[derive(Debug, Default)] +pub struct ColumnIndex { + pub edges: Vec, + pub columns: HashMap>, +} + // Serde: an agent worker cannot write these tables directly, so it posts the // whole manifest to the server, which stores it with the same function the SQL // path uses. @@ -400,6 +467,9 @@ pub struct IngestedNode { pub struct IngestedManifest { pub nodes: Vec, pub edges: Vec<(String, String)>, + /// Column-to-column lineage, when the project asked for it and the engine + /// produced it. Empty is the normal case — see `attach_column_index`. + pub column_edges: Vec, /// The `asset` rows the owning script produces (models) and consumes /// (sources) — what the lineage graph is drawn from. pub assets: Vec, @@ -407,6 +477,88 @@ pub struct IngestedManifest { pub adapter_type: String, } +/// The most column edges one graph stores. +/// +/// A `scan` edge — the column was read to produce the row, not the value — is +/// emitted from every join key and every predicate column to every output +/// column, so one wide model over a multi-column join contributes columns times +/// predicates edges on its own. The cap is what keeps a project shaped like that +/// from turning one deploy into a multi-million-row insert; past it the lineage +/// is truncated and the rest of the graph is unaffected. +pub const MAX_COLUMN_EDGES: usize = 200_000; + +/// Whether the value travelled along this edge, as opposed to the column merely +/// being read to produce the row. +/// +/// A `scan` edge reaches every output column of its model, so it is most of what +/// a wide project's index holds and the first thing `MAX_COLUMN_EDGES` gives up. +/// It is still stored, for a view that wants indirect influence. +pub fn is_direct(lineage_kind: &str) -> bool { + matches!(lineage_kind, "copy" | "mod") +} + +impl IngestedManifest { + /// Fold one `--write-index` pass into the graph. + /// + /// Both halves are scoped to the nodes this graph already kept: the index + /// describes the whole project, while the graph describes what this script's + /// selection builds plus the parents anchoring its edges, and an edge whose + /// endpoint is absent has nothing to draw. + pub fn attach_column_index(&mut self, index: ColumnIndex) { + let kept: std::collections::HashSet<&str> = + self.nodes.iter().map(|n| n.unique_id.as_str()).collect(); + let mut edges: Vec = index + .edges + .into_iter() + .filter(|e| { + kept.contains(e.parent_unique_id.as_str()) + && kept.contains(e.child_unique_id.as_str()) + }) + .collect(); + // Sorted and deduplicated for the digest, which decides whether a run + // stores a snapshot at all: parquet row order is the engine's and two + // passes over one project must not read as two different graphs. + // + // Direct kinds first, so what the truncation below gives up is `scan` — + // the bulk of a wide project's lineage, and the kind that says the column + // was read to produce the row rather than the value. The + // worker's reader already applies this order while decoding, because the + // memory bound has to; repeating it here is what makes the ordering a + // property of the manifest rather than of one caller's reader, and it is + // the only ordering an index assembled some other way would get. + edges.sort_by(|a, b| { + is_direct(&b.lineage_kind) + .cmp(&is_direct(&a.lineage_kind)) + .then_with(|| a.cmp(b)) + }); + edges.dedup(); + edges.truncate(MAX_COLUMN_EDGES); + self.column_edges = edges; + + let mut columns = index.columns; + for node in self.nodes.iter_mut() { + let Some(mut cols) = columns.remove(&node.unique_id) else { + continue; + }; + if cols.is_empty() { + continue; + } + cols.sort_by_key(|c| c.index); + node.column_schema = Some(serde_json::Value::Array( + cols.into_iter() + // A column the analysis typed as nothing still belongs in + // the list — that it exists is the half `manifest.json` + // could not answer. + .map(|c| match c.column_type.is_empty() { + true => serde_json::json!({ "name": c.name }), + false => serde_json::json!({ "name": c.name, "type": c.column_type }), + }) + .collect(), + )); + } + } +} + /// dbt's `materialized` mapped onto Windmill's write strategy. /// /// The mapping is exact for the four strategies Windmill has, and deliberately @@ -657,6 +809,9 @@ pub fn ingest_manifest( .map(|(k, v)| (k.clone(), v.description.clone().unwrap_or_default())) .collect::>()) }), + // Filled by `attach_column_index` when the project asked for it: + // the manifest carries declared columns only. + column_schema: None, freshness: node.freshness.clone(), // The transform the graph renders. Capped: a project can hold // thousands of models and this is duplicated per deploy, so a @@ -828,6 +983,16 @@ pub async fn replace_dbt_manifest( ) .execute(&mut **tx) .await?; + sqlx::query!( + "DELETE FROM dbt_column_edge WHERE workspace_id = $1 AND script_path = $2 + AND script_hash = $3 AND job_id = $4", + workspace_id, + script_path, + script_hash, + job_id + ) + .execute(&mut **tx) + .await?; // The marker, before the rows: a graph with no nodes at all is a legitimate // answer for a dynamic run that disabled every model, and the reader must be // able to tell it from a run that stored nothing. @@ -883,7 +1048,7 @@ async fn insert_graph_rows( "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id, \ resource_type, name, asset_path, materialized, materialize_strategy, unique_key, \ tags, description, test_kind, test_column, test_args, severity, attached_node, \ - columns, freshness, raw_code, original_file_path) ", + columns, column_schema, freshness, raw_code, original_file_path) ", ); q.push_values(chunk, |mut b, n| { b.push_bind(workspace_id) @@ -905,6 +1070,7 @@ async fn insert_graph_rows( .push_bind(&n.severity) .push_bind(&n.attached_node) .push_bind(&n.columns) + .push_bind(&n.column_schema) .push_bind(&n.freshness) .push_bind(&n.raw_code) .push_bind(&n.original_file_path); @@ -928,6 +1094,26 @@ async fn insert_graph_rows( q.push(" ON CONFLICT DO NOTHING"); q.build().execute(&mut **tx).await?; } + + for chunk in ingested.column_edges.chunks(COLUMN_EDGE_INSERT_CHUNK) { + let mut q = sqlx::QueryBuilder::new( + "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id, \ + parent_unique_id, parent_column, child_unique_id, child_column, lineage_kind) ", + ); + q.push_values(chunk, |mut b, e| { + b.push_bind(workspace_id) + .push_bind(script_path) + .push_bind(script_hash) + .push_bind(job_id) + .push_bind(&e.parent_unique_id) + .push_bind(&e.parent_column) + .push_bind(&e.child_unique_id) + .push_bind(&e.child_column) + .push_bind(&e.lineage_kind); + }); + q.push(" ON CONFLICT DO NOTHING"); + q.build().execute(&mut **tx).await?; + } Ok(()) } @@ -968,7 +1154,7 @@ pub async fn replace_dbt_editor_graph( ) -> Result<()> { // By job alone, so re-executing one — a zombie recovered onto another // worker — replaces its rows rather than colliding with them. - for table in ["dbt_node", "dbt_edge", "dbt_graph_snapshot"] { + for table in ["dbt_node", "dbt_edge", "dbt_column_edge", "dbt_graph_snapshot"] { sqlx::query(&format!( "DELETE FROM {table} WHERE workspace_id = $1 AND job_id = $2 AND script_hash IS NULL" )) @@ -1018,7 +1204,7 @@ pub async fn replace_dbt_editor_graph( .fetch_all(&mut **tx) .await?; if !retired.is_empty() { - for table in ["dbt_node", "dbt_edge"] { + for table in ["dbt_node", "dbt_edge", "dbt_column_edge"] { sqlx::query(&format!( "DELETE FROM {table} WHERE workspace_id = $1 AND job_id = ANY($2) \ AND script_hash IS NULL" @@ -1037,6 +1223,8 @@ pub async fn replace_dbt_editor_graph( const NODE_INSERT_CHUNK: usize = 2000; /// Six columns, so the same ceiling allows far more. const EDGE_INSERT_CHUNK: usize = 8000; +/// Nine columns, and by far the most numerous rows of the three. +const COLUMN_EDGE_INSERT_CHUNK: usize = 6000; /// Clear one VERSION's graph: the delete-by-hash route, which only soft-deletes /// its `script` row and so fires no cascade, and the ingest that finds no @@ -1072,6 +1260,15 @@ pub async fn clear_dbt_manifest_version( ) .execute(&mut **tx) .await?; + sqlx::query!( + "DELETE FROM dbt_column_edge + WHERE workspace_id = $1 AND script_path = $2 AND script_hash = $3", + workspace_id, + script_path, + script_hash + ) + .execute(&mut **tx) + .await?; // The marker too, and every job's: a marker left standing for rows that are // gone is read as a snapshot, and its digest still answers the suppression // check — so an identical run would write nothing and then render an empty @@ -1107,7 +1304,7 @@ pub async fn clear_dbt_editor_graphs( workspace_id: &str, script_path: &str, ) -> Result<()> { - for table in ["dbt_node", "dbt_edge", "dbt_graph_snapshot"] { + for table in ["dbt_node", "dbt_edge", "dbt_column_edge", "dbt_graph_snapshot"] { sqlx::query(&format!( "DELETE FROM {table} WHERE workspace_id = $1 AND script_path = $2 AND script_hash IS NULL" @@ -1812,6 +2009,69 @@ mod tests { assert_eq!(back.assets.len(), ingested.assets.len()); assert_eq!(back.assets[0].path, ingested.assets[0].path); } + + // The index describes the whole PROJECT while the graph describes what this + // script's selection builds, so an edge whose endpoint the graph does not + // hold has nothing to draw and must not be stored. + #[test] + fn column_lineage_is_scoped_to_the_nodes_the_graph_kept() { + let mut i = ingested(); + let kept = "model.jaffle_shop.customers"; + let dropped = "model.other_project.elsewhere"; + i.attach_column_index(ColumnIndex { + edges: vec![ + edge("model.jaffle_shop.orders_daily", "id", kept, "id", "copy"), + edge(dropped, "id", kept, "id", "copy"), + edge(kept, "id", dropped, "id", "copy"), + ], + columns: [ + ( + kept.to_string(), + vec![ + col("total", "Float64", 1), + col("id", "Int32", 0), + col("untyped", "", 2), + ], + ), + (dropped.to_string(), vec![col("id", "Int32", 0)]), + ] + .into(), + }); + assert_eq!( + i.column_edges + .iter() + .map(|e| (e.parent_unique_id.as_str(), e.child_unique_id.as_str())) + .collect::>(), + vec![("model.jaffle_shop.orders_daily", kept)] + ); + // In `column_index` order, and a column the analysis could not type still + // belongs to the relation. + assert_eq!( + node(&i, kept).column_schema, + Some(serde_json::json!([ + {"name": "id", "type": "Int32"}, + {"name": "total", "type": "Float64"}, + {"name": "untyped"}, + ])) + ); + assert!(node(&i, "model.jaffle_shop.orders_daily") + .column_schema + .is_none()); + } + + fn edge(from: &str, from_col: &str, to: &str, to_col: &str, kind: &str) -> IngestedColumnEdge { + IngestedColumnEdge { + parent_unique_id: from.into(), + parent_column: from_col.into(), + child_unique_id: to.into(), + child_column: to_col.into(), + lineage_kind: kind.into(), + } + } + + fn col(name: &str, column_type: &str, index: i64) -> IndexedColumn { + IndexedColumn { name: name.into(), column_type: column_type.into(), index } + } } /// Record one model's state for THIS RUN. diff --git a/backend/windmill-common/tests/dbt_graph_storage.rs b/backend/windmill-common/tests/dbt_graph_storage.rs index b6c1a1287a..3226d27e9e 100644 --- a/backend/windmill-common/tests/dbt_graph_storage.rs +++ b/backend/windmill-common/tests/dbt_graph_storage.rs @@ -9,8 +9,8 @@ use sqlx::{Pool, Postgres}; use windmill_common::dbt_manifest::{ clear_dbt_editor_graphs, clear_dbt_manifest_version, clear_dbt_script_state, clear_dbt_script_state_if_path_retired, move_dbt_script_state, prune_dbt_run_graphs, - replace_dbt_editor_graph, replace_dbt_manifest, IngestedManifest, IngestedNode, - DBT_EDITOR_GRAPHS_KEPT, DEPLOYED_GRAPH, DEPLOYED_GRAPH_VERSIONS_KEPT, + replace_dbt_editor_graph, replace_dbt_manifest, IngestedColumnEdge, IngestedManifest, + IngestedNode, DBT_EDITOR_GRAPHS_KEPT, DEPLOYED_GRAPH, DEPLOYED_GRAPH_VERSIONS_KEPT, }; const WS: &str = "test-workspace"; @@ -53,10 +53,36 @@ fn manifest(names: &[&str]) -> IngestedManifest { .windows(2) .map(|w| (format!("model.p.{}", w[0]), format!("model.p.{}", w[1]))) .collect(), + // Same reason: a project that opted into the analysis pass has these, and + // a fixture without them leaves every column-edge insert and sweep in + // this file unexecuted. + column_edges: names + .windows(2) + .map(|w| IngestedColumnEdge { + parent_unique_id: format!("model.p.{}", w[0]), + parent_column: w[0].to_string(), + child_unique_id: format!("model.p.{}", w[1]), + child_column: w[1].to_string(), + lineage_kind: "copy".to_string(), + }) + .collect(), ..Default::default() } } +/// Column edges of one version, so the sweeps can be shown to reach them. +async fn column_edges_for(db: &Pool, hash: i64) -> i64 { + sqlx::query_scalar!( + "SELECT count(*) FROM dbt_column_edge WHERE workspace_id = $1 AND script_hash = $2", + WS, + hash + ) + .fetch_one(db) + .await + .unwrap() + .unwrap_or(0) +} + /// Edges for one version, so a test can assert the batched insert ran at all. async fn edges_for(db: &Pool, hash: i64) -> i64 { sqlx::query_scalar!( @@ -136,6 +162,33 @@ async fn an_identical_run_stores_no_snapshot(db: Pool) { assert_eq!(markers(&db, 1).await, 1, "and leaves no marker of its own"); } +/// A column that is projected AND used as a predicate for the same output column +/// has both a `copy` edge and a `scan` one. They are two facts, and the digest +/// counts both — so the uniqueness key has to carry `lineage_kind`, or the +/// second is dropped by `ON CONFLICT DO NOTHING` while the digest still claims +/// it was stored. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn both_kinds_of_one_column_pair_are_stored(db: Pool) { + deploy_script(&db, 1).await; + let pair = |kind: &str| IngestedColumnEdge { + parent_unique_id: "model.p.a".to_string(), + parent_column: "id".to_string(), + child_unique_id: "model.p.b".to_string(), + child_column: "id".to_string(), + lineage_kind: kind.to_string(), + }; + let mut m = manifest(&["a", "b"]); + m.column_edges = vec![pair("copy"), pair("scan")]; + + let mut tx = db.begin().await.unwrap(); + replace_dbt_manifest(&mut tx, WS, PATH, 1, None, &m, "root") + .await + .unwrap(); + tx.commit().await.unwrap(); + + assert_eq!(column_edges_for(&db, 1).await, 2, "both kinds survive"); +} + /// A run whose model set differs keeps its own, and the version's is untouched: /// this is what lets an older run page render the project that run built. #[sqlx::test(migrations = "../migrations", fixtures("base"))] @@ -280,6 +333,8 @@ async fn deleting_the_script_cascades_to_every_sidecar(db: Pool) { assert_eq!(nodes_for(&db, 2, DEPLOYED_GRAPH).await, 0); assert_eq!(edges_for(&db, 1).await, 0); assert_eq!(edges_for(&db, 2).await, 0); + assert_eq!(column_edges_for(&db, 1).await, 0); + assert_eq!(column_edges_for(&db, 2).await, 0); assert_eq!(markers_for_path(&db).await, 0); } @@ -320,7 +375,7 @@ async fn the_sweep_takes_old_snapshots_and_spares_the_version(db: Pool tx.commit().await.unwrap(); // Age one snapshot past the window, rows and marker together. - for t in ["dbt_node", "dbt_edge", "dbt_graph_snapshot"] { + for t in ["dbt_node", "dbt_edge", "dbt_column_edge", "dbt_graph_snapshot"] { sqlx::query(&format!( "UPDATE {t} SET ingested_at = now() - interval '400 days' WHERE job_id = $1" )) diff --git a/backend/windmill-worker/Cargo.toml b/backend/windmill-worker/Cargo.toml index e75afc053e..5e95a1e20a 100644 --- a/backend/windmill-worker/Cargo.toml +++ b/backend/windmill-worker/Cargo.toml @@ -114,6 +114,10 @@ rust_decimal.workspace = true jsonwebtoken.workspace = true sha2.workspace = true hmac.workspace = true +# Reads the dbt engine's `target/index/*.parquet`, which is where column-level +# lineage lives. Unconditional rather than behind the `parquet` FEATURE: that one +# is object storage, and a build without it still runs dbt jobs. +parquet.workspace = true pem = { workspace = true, optional = true } rsa = { workspace = true, optional = true } urlencoding.workspace = true diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index b3324f87ba..b577097c9d 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -408,16 +408,6 @@ pub fn create_empty_dir(path: &PathBuf) -> std::io::Result<()> { } } -/// Signals a detached `spawn_blocking` task that the future awaiting it is -/// gone, so it can stop instead of running to completion in the background. -struct AbortOnDrop(std::sync::Arc); - -impl Drop for AbortOnDrop { - fn drop(&mut self) { - self.0.store(true, std::sync::atomic::Ordering::Relaxed); - } -} - /// Lay down the tree of an app-backed repository, which git can't clone /// because its URL carries no credential. /// @@ -490,7 +480,7 @@ async fn fetch_repo_archive( // stopping it, so the flag is what a cancelled job uses to reach the // extraction loop. The guard sets it when this future is dropped. let aborted = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); - let _abort_on_drop = AbortOnDrop(aborted.clone()); + let _abort_on_drop = crate::common::AbortOnDrop(aborted.clone()); let unpack_archive = download_archive.clone(); tokio::task::spawn_blocking(move || { unpack_repo_archive(&unpack_archive, &download_target, &aborted) diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index f6391eab07..0a5deb52d2 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -67,6 +67,20 @@ mount { #[cfg(not(debug_assertions))] pub const DEV_CONF_NSJAIL: &str = ""; +/// Tells a `spawn_blocking` task to stop when the future awaiting it goes away. +/// +/// Dropping a `JoinHandle` detaches the task rather than cancelling it, so a +/// cancelled or timed-out phase otherwise leaves the blocking pool working on an +/// answer nobody will read. Hold one of these beside the handle and have the +/// blocking loop check the flag. +pub(crate) struct AbortOnDrop(pub(crate) std::sync::Arc); + +impl Drop for AbortOnDrop { + fn drop(&mut self) { + self.0.store(true, std::sync::atomic::Ordering::Relaxed); + } +} + /// Turn a JSON value into the string a shell/CLI arg should receive: a JSON string /// becomes its inner value, anything else is re-serialized compactly. pub(crate) fn raw_to_string(x: &str) -> String { diff --git a/backend/windmill-worker/src/dbt_column_index.rs b/backend/windmill-worker/src/dbt_column_index.rs new file mode 100644 index 0000000000..ab5853b9b6 --- /dev/null +++ b/backend/windmill-worker/src/dbt_column_index.rs @@ -0,0 +1,589 @@ +//! Column-level lineage and real column schemas, from the engine's own static +//! analysis. +//! +//! `manifest.json` carries neither. What does is the parquet index an engine +//! writes under `dbt compile --static-analysis strict --write-index`: +//! `dbt.column_lineage.parquet` (column-to-column edges, each labelled `copy`, +//! `mod` or `scan`) and `dbt.node_columns.parquet` (every column of every node, +//! typed and ordered, rather than only the ones an author documented). +//! +//! Four properties shape everything here, all of them measured against the real +//! engines rather than assumed: +//! +//! - **Strict analysis rejects SQL the default accepts.** An unresolvable +//! identifier is an error under `strict` and compiles fine otherwise, so this +//! is a SEPARATE pass with its own `--target-path`, never a flag on the build, +//! and it is opt-in per project. +//! - **A failed pass still writes the index**, holding every edge of the models +//! that did analyze. So the artifact is read whatever the exit status. +//! - **The flag is not the capability.** `dbt-core` 2.0.0-alpha.5 accepts +//! `--write-index`, declares the views over these two tables in its own +//! `views.sql`, and writes neither file; only Fusion does today. Nothing here +//! asks which engine it is beyond "has the flag" — a release that starts +//! writing them is picked up with no change. +//! - **An incremental model has two shapes, and one ingest holds one of them.** +//! `is_incremental()` is false when the target does not exist or the build +//! is `--full-refresh`, so the `{{ this }}` self-join — and any `ref()` inside +//! that branch — compiles only in the other case. What this stores is +//! therefore what the compile in front of it saw: at DEPLOY, before the first +//! build, that is the cold shape, and a project deployed again after its +//! tables exist stores the incremental one for the same source. Nothing here +//! can reconcile that; dbt has no mode that emits both. The flag is taken from +//! the build so a per-run ingest matches its own run, and the version's graph +//! is honest about the compile that produced it rather than about every run +//! that will follow. + +use std::collections::HashSet; +use std::ops::ControlFlow; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::path::Path; +use std::time::Duration; + +use parquet::file::reader::{FileReader, SerializedFileReader}; +use parquet::record::{Field, Row}; +use uuid::Uuid; +use windmill_common::dbt_manifest::{ + is_direct, ColumnIndex, IndexedColumn, IngestedColumnEdge, MAX_COLUMN_EDGES, +}; +use windmill_common::error; +use windmill_common::worker::Connection; +use windmill_parser_yaml::dbt::DbtDescriptor; +use windmill_queue::append_logs; + +use crate::dbt_executor::{dbt_command, Invocation, PreparedProject}; +use crate::handle_child::JobCtx; + +/// Where the lineage pass writes, relative to the project directory. +/// +/// Its own tree, not the runtime's `wm_target`: a `dbt compile` writes +/// `manifest.json` and `run_results.json` like any other invocation, and after a +/// build those two are what the graph ingest and `dbt retry` read. +const CLL_ARTIFACTS_DIR: &str = "wm_target_cll"; + +const COLUMN_LINEAGE_PARQUET: &str = "dbt.column_lineage.parquet"; +const NODE_COLUMNS_PARQUET: &str = "dbt.node_columns.parquet"; + +/// Run the lineage pass and read what it produced. +/// +/// Two steps with deliberately different contracts, because conflating them is +/// what made a best-effort annotation able to fail the job it annotates: +/// +/// - [`compile_index`] runs a subprocess and owns the JOB's semantics. Only a +/// cancellation or the job's own deadline can `Err` out of it; a non-zero exit +/// and an over-long output are outcomes, not failures. +/// - [`read_index`] owns the ARTIFACT's semantics. Reading it never fails the +/// job on the artifact's account: an absent, unreadable or partial index is a +/// value, not an error. It runs UNDER the poller all the same, so the job can +/// still end the phase — a cancel, a completion or the phase timeout — which +/// is the job's semantics reaching in, not the artifact's reaching out. +/// +/// The phase budget wraps the compile alone, because it exists to leave the +/// BUILD its share of the clock and only the compile can spend that share +/// unboundedly. The decode's own end is the job's: the poller it runs under +/// stops it when the job stops. +pub(crate) async fn collect( + p: &PreparedProject, + descriptor: &DbtDescriptor, + inv: &Invocation, + // The dbt subcommand the job runs, which decides the effective + // `--full-refresh` — see `dbt_executor::full_refresh`. + command: &str, + ctx: &mut JobCtx<'_>, + job_id: &Uuid, + w_id: &str, + conn: &Connection, + kept: &HashSet<&str>, +) -> error::Result> { + if !descriptor.column_lineage { + return Ok(None); + } + if !p.engine.engine.writes_column_index() { + append_logs( + job_id, + w_id, + format!( + "\n`column_lineage` is set, but the {} engine has no `--write-index`: column \ + lineage needs an engine that does static analysis. The rest of the graph is \ + unaffected.\n", + p.engine.engine.as_str() + ), + conn, + ) + .await; + return Ok(None); + } + + let index_dir = p.project_dir.join(CLL_ARTIFACTS_DIR).join("index"); + let Some(compiled) = compile_index(p, descriptor, inv, command, ctx, job_id, w_id, conn).await? else { + return Ok(None); + }; + let coverage = Coverage::of(&compiled); + + // Decoded UNDER the poller, not followed by a check of its own. The decode is + // the one phase of this pass with no subprocess behind it, so nothing else + // heartbeats while it runs: left alone, a large index is a silent worker for + // as long as it takes, which the zombie sweep reads as a dead job and + // restarts. The poller pings throughout and ends this with an `Err` if the + // job is cancelled or completed meanwhile — the job's own semantics, which + // this module may always propagate. + let artifact = crate::handle_child::run_future_with_polling_update_job_poller( + *job_id, + ctx.timeout(), + conn, + ctx.mem_peak, + ctx.canceled_by, + async { Ok(read_index(&index_dir, kept).await) }, + ctx.worker_name, + w_id, + &mut Some(ctx.occupancy_metrics), + Box::pin(futures::stream::empty()), + ) + .await?; + + // What only the pass knows. The COUNTS are logged where the index is folded + // into the graph, since the graph decides how much of it is kept. + let note = match artifact { + Artifact::Read(index) => { + if let Some(note) = coverage.caveat() { + log(job_id, w_id, note, &compiled.stderr, conn).await; + } + return Ok(Some(index)); + } + // The truncated arms come first: a compile stopped part-way explains an + // absent or unreadable artifact, and blaming the engine's capability + // for it sends the reader to check the wrong thing entirely. + Artifact::Missing if matches!(coverage, Coverage::Truncated) => format!( + "No column lineage: the analysis pass printed more than this runtime reads and was \ + stopped before it wrote `{COLUMN_LINEAGE_PARQUET}`." + ), + Artifact::Unreadable(why) if matches!(coverage, Coverage::Truncated) => format!( + "No column lineage: the analysis pass was stopped for printing more than this \ + runtime reads, and the `{COLUMN_LINEAGE_PARQUET}` it had written could not be read \ + ({why})." + ), + // Said apart from the one below, because it sends the reader somewhere + // else entirely: the engine did its job and this runtime could not read + // what it wrote. + Artifact::Unreadable(why) => format!( + "No column lineage: `{COLUMN_LINEAGE_PARQUET}` was written but could not be read \ + ({why}). The graph is unaffected." + ), + Artifact::Missing => format!( + "No column lineage: the analysis pass wrote no `{COLUMN_LINEAGE_PARQUET}`. Only an \ + engine that computes it does, and only for the warehouses it analyzes natively — \ + the flag alone is not the capability." + ), + }; + // The engine's own diagnostics come along. They are how a reader learns that + // this adapter turned static analysis off, which it reports as a warning on + // a SUCCESSFUL compile that nothing else would show. + log(job_id, w_id, ¬e, &compiled.stderr, conn).await; + Ok(None) +} + +async fn log(job_id: &Uuid, w_id: &str, note: &str, stderr: &str, conn: &Connection) { + append_logs( + job_id, + w_id, + format!("\n{note}\n{}", diagnostics(stderr)), + conn, + ) + .await; +} + +/// How completely the analysis compile covered the project. +/// +/// Every way the COMPILE can disappoint is a value here rather than an error. An +/// `Err` from `compile_index` is the JOB's — a cancellation or its deadline — +/// and must fail it; the pass giving up on its own terms is `Ok(None)` and has +/// already been logged. +enum Coverage { + /// Every model analyzed. + Whole, + /// `--static-analysis strict` rejected part of the project. Whatever it did + /// analyze is still in the index. + Partial, + /// The output ceiling killed the compile mid-run. Distinct from `Partial`: + /// nothing rejected the project, but the index is however far it had got, so + /// it is not `Whole` either. + Truncated, +} + +impl Coverage { + fn of(c: &crate::dbt_executor::Captured) -> Self { + match (c.truncated, c.success) { + (true, _) => Coverage::Truncated, + (false, true) => Coverage::Whole, + (false, false) => Coverage::Partial, + } + } + + /// What to tell the reader when an index WAS produced. `None` for a run that + /// covered everything, which needs no caveat. + fn caveat(&self) -> Option<&'static str> { + match self { + Coverage::Whole => None, + Coverage::Partial => Some( + "Column lineage: `--static-analysis strict` rejected part of the project, so \ + the lineage covers only the models it could analyze.", + ), + Coverage::Truncated => Some( + "Column lineage: the analysis pass printed more than this runtime reads and was \ + stopped, so the lineage covers only the models it had reached.", + ), + } + } +} + +/// Run `dbt compile --static-analysis strict --write-index`, under this phase's +/// share of the job's clock. +/// +/// `Ok(None)` is "the pass gave up and said so"; `Err` is the job's own +/// cancellation or deadline and must propagate. Nothing outlives this function: +/// the budget is a race around the child, and dropping that future kills it +/// through `run_captured`'s `kill_on_drop`. +async fn compile_index( + p: &PreparedProject, + descriptor: &DbtDescriptor, + inv: &Invocation, + command: &str, + ctx: &mut JobCtx<'_>, + job_id: &Uuid, + w_id: &str, + conn: &Connection, +) -> error::Result> { + // A previous pass in the same job directory — a retry's second attempt — + // would otherwise be read back as this one's answer. + tokio::fs::remove_dir_all(p.project_dir.join(CLL_ARTIFACTS_DIR)) + .await + .ok(); + + let mut cmd = dbt_command( + p, + &[ + "compile", + "--static-analysis", + "strict", + "--write-index", + // Documented as what builds the CLL graph, and `--write-index` alone + // happens to imply it on the engine probed. Passed explicitly so the + // pass does not depend on which of the two is doing the work. + "--write-lineage", + "--target-path", + CLL_ARTIFACTS_DIR, + ], + ); + // The flag already wins over the env var dbt_command sets, but setting both + // means this pass cannot write into the runtime's artifacts even if that + // precedence ever changes — and what is in there after a build is the + // `run_results.json` a `dbt retry` resumes from. + cmd.env("DBT_TARGET_PATH", CLL_ARTIFACTS_DIR); + crate::dbt_executor::add_vars(&mut cmd, descriptor, inv)?; + // The BUILD's answer, not the descriptor's default: `is_incremental()` + // branches on it, so a model reading `{{ this }}` compiles its self-join — + // and any `ref()` inside that branch — only when this is absent. Guessing + // here stores lineage for SQL the run never executed. + if crate::dbt_executor::full_refresh(descriptor, inv, command)? { + cmd.arg("--full-refresh"); + } + // Captured rather than streamed: a strict-analysis failure is a wall of + // diagnostics about SQL the build itself accepts, and this pass decides + // nothing about whether that build runs. + // Read before the future below borrows `ctx` mutably. + let budget = phase_budget(ctx); + let run = crate::dbt_executor::run_captured( + cmd, + "dbt compile (column lineage)", + ctx, + job_id, + w_id, + conn, + CLL_MAX_OUTPUT_BYTES, + // The ceiling is this pass's, not the job's: a compile that prints more + // than it than we care to read has still analyzed the project, and the + // index it wrote is on disk either way. + crate::dbt_executor::Overflow::Truncate, + ); + let Some(budget) = budget else { + return Ok(Some(run.await?)); + }; + match tokio::time::timeout(budget, run).await { + Ok(r) => Ok(Some(r?)), + Err(_) => { + append_logs( + job_id, + w_id, + format!( + "\nNo column lineage: the analysis pass did not finish within {}s, half of \ + what was left of this job's time. The build below gets the rest.\n", + budget.as_secs() + ), + conn, + ) + .await; + Ok(None) + } + } +} + +/// stdout the pass may produce. It is a compile, so this is diagnostics rather +/// than data. +const CLL_MAX_OUTPUT_BYTES: usize = 1 << 20; + +/// The share of the job's remaining wall clock this pass may spend. +/// +/// A per-run refresh ingests BEFORE the build and shares the job's one deadline, +/// so an unbounded pass on a slow project would hand `dbt build` an expired +/// budget and fail the run it exists only to annotate. Half leaves the build at +/// least as long as the annotation was allowed to take. +/// +/// Spent as a race around the COMPILE rather than as a shortened deadline handed +/// to the runner: the runner reports its expiry as an `Err`, indistinguishable +/// from a cancellation or the job's own deadline, and those two MUST fail the +/// job. Expiring here is this budget and nothing else. The child dies with the +/// dropped future through `run_captured`'s `kill_on_drop`; the decode is outside +/// this race and answers to the poller instead. +fn phase_budget(ctx: &JobCtx<'_>) -> Option { + ctx.timeout() + .map(|left| Duration::from_secs((left.max(0) as u64 / 2).max(1))) +} + +/// The tail of what the engine said, bounded. The whole of it is every rendered +/// model on a large project, which is not what a job log is for. +const DIAGNOSTIC_LINES: usize = 40; + +fn diagnostics(out: &str) -> String { + let lines: Vec<&str> = out.lines().filter(|l| !l.trim().is_empty()).collect(); + let tail = &lines[lines.len().saturating_sub(DIAGNOSTIC_LINES)..]; + match tail.is_empty() { + true => String::new(), + false => format!("{}\n", tail.join("\n")), + } +} + +/// What came back from the artifact. Never an `Err`: nothing the file does or +/// fails to do is a reason to fail a job. `Unreadable` is separate from +/// `Missing` because the two send a reader looking in different places — one at +/// their engine and adapter, the other at a file that exists. +enum Artifact { + Read(ColumnIndex), + Missing, + Unreadable(String), +} + +/// Read both parquets, if the lineage one is there. +/// +/// The column schemas alone are not worth a graph: they arrive with the lineage +/// or not at all, and a node's declared columns already answer for the case +/// where the pass never ran. +async fn read_index(index_dir: &Path, kept: &HashSet<&str>) -> Artifact { + let lineage = index_dir.join(COLUMN_LINEAGE_PARQUET); + if !tokio::fs::try_exists(&lineage).await.unwrap_or(false) { + return Artifact::Missing; + } + let columns = index_dir.join(NODE_COLUMNS_PARQUET); + // Owned, because the decode moves to a blocking thread. The index describes + // the whole project while this graph describes one selection of it, so + // scoping HERE is what keeps the bound below from being spent on rows the + // graph would discard anyway. + let kept: HashSet = kept.iter().map(|s| (*s).to_string()).collect(); + // Dropping the handle of a blocking task does NOT stop it: the poller + // cancelling this phase would otherwise leave a thread decoding millions of + // rows for a job that is over. `abandoned` is set when this future is + // dropped, and the row loop reads it. + let abandoned = Arc::new(AtomicBool::new(false)); + let _stop = crate::common::AbortOnDrop(abandoned.clone()); + // Decompressing and decoding a parquet is CPU work on a file the engine just + // wrote, so it does not belong on the runtime's poll thread. + let read = tokio::task::spawn_blocking(move || { + read_index_blocking(&lineage, &columns, &kept, &abandoned) + }) + .await; + match read { + Ok(Ok(index)) => Artifact::Read(index), + Ok(Err(e)) => Artifact::Unreadable(e.to_string()), + Err(e) => Artifact::Unreadable(e.to_string()), + } +} + +fn read_index_blocking( + lineage: &Path, + columns: &Path, + kept: &HashSet, + abandoned: &AtomicBool, +) -> error::Result { + let mut out = ColumnIndex::default(); + // ONE pass, with the two kinds bucketed as they arrive. `copy` and `mod` say + // the value itself travelled, so they get the whole budget; `scan` — the + // column was read to produce the ROW, which reaches every output column of + // its model and is the bulk of a wide project's index — fills only what is + // left over at the end. Reading the file twice to get that ordering would + // double the decode of exactly the large index this bound exists for. + let mut scan: Vec = Vec::new(); + for_each_row(lineage, abandoned, |row| { + let lineage_kind = string(row, "lineage_kind"); + let parent_unique_id = string(row, "from_node_unique_id"); + let child_unique_id = string(row, "to_node_unique_id"); + let parent_column = string(row, "from_column_name"); + let child_column = string(row, "to_column_name"); + // A column of a node the analysis could not name is not an endpoint the + // graph can draw, and neither is one outside this graph's nodes. + if parent_column.is_empty() + || child_column.is_empty() + || !kept.contains(&parent_unique_id) + || !kept.contains(&child_unique_id) + { + return ControlFlow::Continue(()); + } + let edge = IngestedColumnEdge { + parent_unique_id, + parent_column, + child_unique_id, + child_column, + lineage_kind, + }; + // The bound covers BOTH buckets, so the pass never holds more than one + // budget's worth however the kinds are distributed. + let held = out.edges.len() + scan.len(); + if is_direct(&edge.lineage_kind) { + // A direct edge displaces a `scan` one: the budget is spent on + // value flow first. + if held >= MAX_COLUMN_EDGES { + scan.pop(); + } + out.edges.push(edge); + // The edge that FILLS the budget ends the read, not the next one to + // arrive: once the displacing kind is full nothing later in the file + // can be kept, and waiting for another direct edge to say so decodes + // a `scan`-only tail all the way to the backstop for nothing. + return match out.edges.len() >= MAX_COLUMN_EDGES { + true => ControlFlow::Break(()), + false => ControlFlow::Continue(()), + }; + } + if held < MAX_COLUMN_EDGES { + scan.push(edge); + } + // Not a stopping point even when full: a direct edge still to come takes + // a `scan` entry's place. + ControlFlow::Continue(()) + })?; + out.edges.append(&mut scan); + // Absent is normal — an engine can write the lineage table and not this one — + // and unreadable is not worth losing the lineage over. + let mut held = 0usize; + let _ = for_each_row(columns, abandoned, |row| { + let unique_id = string(row, "unique_id"); + let name = string(row, "column_name"); + if held >= MAX_INDEXED_COLUMNS { + return ControlFlow::Break(()); + } + if name.is_empty() || !kept.contains(&unique_id) { + return ControlFlow::Continue(()); + } + held += 1; + // The author's `data_type` where `schema.yml` gives one, since that is + // what the project calls the column; the analysis's own inference + // otherwise. + let column_type = match string(row, "declared_type") { + t if !t.is_empty() => t, + _ => string(row, "inferred_type"), + }; + out.columns + .entry(unique_id) + .or_default() + .push(IndexedColumn { + name, + column_type, + index: int(row, "column_index").unwrap_or(i64::MAX), + }); + ControlFlow::Continue(()) + }); + Ok(out) +} + +/// The most rows of `dbt.node_columns.parquet` one pass keeps. One per column of +/// the project, so the same bound as the edges is far more than any project +/// reaches; it exists for the same reason. +const MAX_INDEXED_COLUMNS: usize = MAX_COLUMN_EDGES; + +/// The most rows of an index one pass DECODES, whatever it keeps of them. +/// +/// A bound on work rather than on memory, and the two are separate because the +/// input this defends against is the one that cannot be collected: `scan` +/// lineage is emitted from every predicate and join column to every output +/// column, so a project shaped that way writes an index whose row count is +/// quadratic in its widest model. This pass runs outside the phase budget, on a +/// blocking thread, and nothing the file contains may fail a deploy or a run — +/// so the file it walks needs an end even when almost nothing in it is +/// retained. The abandonment flag ends it sooner when the job is over; this is +/// the bound for a job that is not. +const MAX_INDEX_ROWS: usize = 4_000_000; + +/// Decode a parquet a row at a time, handing each to `f` and never holding two. +/// +/// Collecting first would put a `Vec` — each row carrying its own copy of +/// every column NAME — in front of the caller's own bound, which is what would +/// take the worker process down on the index described above. +/// +/// `f` says when it has all it will take, and that is the ordinary end: this +/// runs outside the phase budget, so every row decoded past the point of being +/// able to keep one is wall clock the build below does not get. +fn for_each_row( + path: &Path, + abandoned: &AtomicBool, + mut f: impl FnMut(&Row) -> ControlFlow<()>, +) -> error::Result<()> { + let fail = |e: parquet::errors::ParquetError| { + error::Error::internal_err(format!("reading {}: {e}", path.display())) + }; + let file = std::fs::File::open(path) + .map_err(|e| error::Error::internal_err(format!("opening {}: {e}", path.display())))?; + let reader = SerializedFileReader::new(file).map_err(fail)?; + for (n, row) in reader.get_row_iter(None).map_err(fail)?.enumerate() { + // Nobody is waiting for this any more — the job was cancelled, completed + // or ran out of time while it decoded. + if abandoned.load(Ordering::Relaxed) { + break; + } + if n >= MAX_INDEX_ROWS { + tracing::warn!( + "dbt column index: {} holds more than {MAX_INDEX_ROWS} rows; the rest is dropped", + path.display() + ); + break; + } + if f(&row.map_err(fail)?).is_break() { + break; + } + } + Ok(()) +} + +/// By NAME, not by position: these tables are the engine's own schema and it +/// adds columns to them between releases. +fn field<'a>(row: &'a Row, name: &str) -> Option<&'a Field> { + row.get_column_iter() + .find(|(k, _)| k.as_str() == name) + .map(|(_, v)| v) +} + +fn string(row: &Row, name: &str) -> String { + match field(row, name) { + Some(Field::Str(s)) => s.clone(), + Some(Field::Bytes(b)) => String::from_utf8_lossy(b.data()).into_owned(), + _ => String::new(), + } +} + +fn int(row: &Row, name: &str) -> Option { + match field(row, name) { + Some(Field::Long(v)) => Some(*v), + Some(Field::Int(v)) => Some(*v as i64), + Some(Field::Short(v)) => Some(*v as i64), + Some(Field::UInt(v)) => Some(*v as i64), + Some(Field::ULong(v)) => i64::try_from(*v).ok(), + _ => None, + } +} diff --git a/backend/windmill-worker/src/dbt_executor.rs b/backend/windmill-worker/src/dbt_executor.rs index 727ed78c5c..bb449f1810 100644 --- a/backend/windmill-worker/src/dbt_executor.rs +++ b/backend/windmill-worker/src/dbt_executor.rs @@ -491,7 +491,7 @@ pub(crate) async fn handle_dbt_job( // For a retry the restored manifest already describes the invocation // being resumed, so only the ingest runs — with that invocation's // arguments, which the selection resolver needs to interpolate. - ingest_from_run(&prepared, &descriptor, &inv, &mut ctx, job, conn).await?; + ingest_from_run(&prepared, &descriptor, &inv, &command, &mut ctx, job, conn).await?; } // A read-only command prints rows to stdout, so it is captured rather than @@ -826,12 +826,26 @@ pub(crate) async fn dbt_dep( None => GraphPublisher::Unversioned, }; let superseded = if let Some(warehouse) = prepared.warehouse.as_deref() { - let ingested = windmill_common::dbt_manifest::ingest_manifest( + let mut ingested = windmill_common::dbt_manifest::ingest_manifest( &manifest, warehouse, prepared.default_database.as_deref(), selected.as_ref(), ); + attach_column_index( + &mut ingested, + &prepared, + &descriptor, + &inv, + // A deploy resolves the project by parsing it; nothing is built, so + // the pass takes the descriptor's own answer. + "parse", + &mut ctx, + job_id, + w_id, + &conn, + ) + .await?; let published = persist_ingest( db, w_id, @@ -1040,6 +1054,9 @@ impl GraphRefresh { if selection_is_overridden(descriptor, args)? { self.per_run_models = true; } + if full_refresh_is_overridden(descriptor, args)? { + self.per_run_models = true; + } Ok(()) } } @@ -2507,8 +2524,7 @@ async fn run_dbt( if let Some(t) = descriptor.threads { cmd.args(["--threads", &t.to_string()]); } - let full_refresh = arg_bool(&inv.args, "full_refresh")?.unwrap_or(descriptor.full_refresh); - if full_refresh && command != "test" { + if full_refresh(descriptor, inv, command)? { cmd.arg("--full-refresh"); } } @@ -3160,7 +3176,8 @@ async fn run_show( conn, SHOW_MAX_OUTPUT_BYTES, ) - .await?; + .await? + .stdout; // dbt frames the rows as `{"node": …, "show": [ … ]}`, pretty-printed, with a // banner before and a deprecation summary after — so neither "the line starting // with `{`" nor "first `{` to the end" parses. A streaming deserializer stops at @@ -3343,7 +3360,7 @@ async fn run_parse_only( // manifest and the selection while the warehouse only keys them — so a project // with no warehouse identity still reports what dbt found. The placeholder // reaches no row: the guard below returns before anything is written. - let ingested = windmill_common::dbt_manifest::ingest_manifest( + let mut ingested = windmill_common::dbt_manifest::ingest_manifest( &manifest, p.warehouse.as_deref().unwrap_or("unkeyed"), p.default_database.as_deref(), @@ -3363,6 +3380,20 @@ async fn run_parse_only( else { return Ok(to_raw_value(&result)); }; + // AFTER the guard: the pass is a second `dbt compile` and a parquet decode, + // and a parse that stores nothing has nowhere to put what it would produce. + attach_column_index( + &mut ingested, + p, + descriptor, + inv, + "parse", + ctx, + &job.id, + &job.workspace_id, + conn, + ) + .await?; match conn { Connection::Sql(db) => match job.runnable_id.map(|h| h.0) { Some(script_hash) => { @@ -3422,11 +3453,70 @@ async fn run_parse_only( Ok(to_raw_value(&result)) } +/// Fold this project's column lineage into the graph about to be stored, when +/// the descriptor asked for it. +/// +/// One helper for all three ingests — deploy, editor parse, per-run refresh — +/// because a graph that carries column lineage in one provenance and not another +/// reads as the lineage having disappeared. +async fn attach_column_index( + ingested: &mut windmill_common::dbt_manifest::IngestedManifest, + p: &PreparedProject, + descriptor: &DbtDescriptor, + inv: &Invocation, + command: &str, + ctx: &mut JobCtx<'_>, + job_id: &Uuid, + w_id: &str, + conn: &Connection, +) -> error::Result<()> { + // The nodes this graph kept, so the pass reads only rows it could store: the + // index describes the whole project, this graph one selection of it. + let kept: std::collections::HashSet<&str> = ingested + .nodes + .iter() + .map(|n| n.unique_id.as_str()) + .collect(); + let index = + crate::dbt_column_index::collect( + p, descriptor, inv, command, ctx, job_id, w_id, conn, &kept, + ) + .await?; + drop(kept); + let Some(index) = index else { + return Ok(()); + }; + let found = index.edges.len(); + ingested.attach_column_index(index); + let kept = ingested.column_edges.len(); + let typed: usize = ingested + .nodes + .iter() + .filter(|n| n.column_schema.is_some()) + .count(); + // Counted here rather than at the pass: the index describes the whole + // project and this graph describes one selection of it, so `found` is what + // dbt produced and `kept` is what the graph can draw. + let dropped = match found.saturating_sub(kept) { + 0 => String::new(), + n => format!(" ({n} outside this graph or past the cap)"), + }; + append_logs( + job_id, + w_id, + format!("\nIngested {kept} column lineage edges{dropped} and typed {typed} nodes\n"), + conn, + ) + .await; + Ok(()) +} + /// Refresh the stored graph from the manifest this run produced. async fn ingest_from_run( p: &PreparedProject, descriptor: &DbtDescriptor, inv: &Invocation, + command: &str, ctx: &mut JobCtx<'_>, job: &MiniPulledJob, conn: &Connection, @@ -3443,12 +3533,24 @@ async fn ingest_from_run( // filter this run's manifest by a different node set than it built. let selected = resolve_selection(p, descriptor, inv, ctx, &job.id, &job.workspace_id, conn).await?; - let ingested = windmill_common::dbt_manifest::ingest_manifest( + let mut ingested = windmill_common::dbt_manifest::ingest_manifest( &manifest, warehouse, p.default_database.as_deref(), selected.as_ref(), ); + attach_column_index( + &mut ingested, + p, + descriptor, + inv, + command, + ctx, + &job.id, + &job.workspace_id, + conn, + ) + .await?; // Only a run whose models are its own snapshots per run. A static // descriptor at a moved profile re-ingests the VERSION's graph, since the // move outlives the run; one that neither drifted nor overrode anything @@ -3773,7 +3875,9 @@ async fn resolve_selection( // through the job-log writer, which `NO_LOGS_AT_ALL` discards — the selection // would resolve to the empty set and the ingest would wipe the script's assets // while dbt went on building the descriptor's models. - let stdout = run_capturing(cmd, "dbt ls", ctx, job_id, w_id, conn, LS_MAX_OUTPUT_BYTES).await?; + let stdout = run_capturing(cmd, "dbt ls", ctx, job_id, w_id, conn, LS_MAX_OUTPUT_BYTES) + .await? + .stdout; let mut set = std::collections::HashSet::new(); for line in stdout.lines() { let line = line.trim(); @@ -3826,6 +3930,34 @@ async fn resolve_selection( /// what is kept is the TAIL, because dbt prints its error summary last. const CAPTURE_MAX_STDERR_BYTES: usize = 64 * 1024; +/// What a captured invocation produced. `stderr` is where dbt writes its +/// diagnostics — the errors and warnings block — so a caller that has to explain +/// a SUCCESSFUL run needs it as much as a failing one does. +pub(crate) struct Captured { + pub stdout: String, + pub stderr: String, + /// Whether the child exited zero. Separate from the `Result` on purpose: an + /// `Err` from `run_captured` is the JOB's — a cancellation or its deadline — + /// so a caller that tolerates a failed command must still propagate one. + pub success: bool, + /// Whether the output ceiling cut the child short. Only ever true under + /// [`Overflow::Truncate`]. + pub truncated: bool, +} + +/// What an over-long stdout means to the caller. +/// +/// The ceiling belongs to the PASS, not to the job: a caller that only annotates +/// a job wants to keep what it read and carry on, while one whose whole result +/// is that output has nothing to return without it. +#[derive(PartialEq, Eq, Clone, Copy)] +pub(crate) enum Overflow { + /// Fail the job. For a command whose output IS the answer. + Fail, + /// Stop reading, kill the child, and report `truncated`. + Truncate, +} + /// Run a command for its stdout under the job's cancellation and timeout. /// The same poller `handle_child` uses drives them, so a cancel or a deadline /// drops the wait future — which owns the child, and `kill_on_drop` then @@ -3838,7 +3970,7 @@ const CAPTURE_MAX_STDERR_BYTES: usize = 64 * 1024; /// never holds more than it, so it has to be enforced while reading. Both pipes /// are drained concurrently because a child that fills the one nobody reads /// blocks forever. -async fn run_capturing( +pub(crate) async fn run_captured( mut cmd: Command, name: &str, ctx: &mut JobCtx<'_>, @@ -3846,7 +3978,8 @@ async fn run_capturing( w_id: &str, conn: &Connection, max_stdout_bytes: usize, -) -> error::Result { + on_overflow: Overflow, +) -> error::Result { use tokio::io::AsyncReadExt; let mut child = cmd @@ -3881,6 +4014,7 @@ async fn run_capturing( let mut out_buf = vec![0u8; 16 * 1024]; let mut err_buf = vec![0u8; 16 * 1024]; let (mut out_open, mut err_open) = (true, true); + let mut truncated = false; while out_open || err_open { tokio::select! { r = stdout_pipe.read(&mut out_buf[..]), if out_open => match r { @@ -3888,14 +4022,19 @@ async fn run_capturing( Ok(n) => { if stdout.len() + n > max_stdout_bytes { // Killed here rather than left to `kill_on_drop` - // so the child is gone before the error unwinds, - // not merely once this future is dropped. + // so the child is gone before this returns, not + // merely once the future is dropped. let _ = child.kill().await; - return Err(Error::ExecutionErr(format!( - "{name} produced more than {} MB of output. Narrow the \ - selection, or query the relation from a SQL script.", - max_stdout_bytes / 1024 / 1024 - ))); + if on_overflow == Overflow::Fail { + return Err(Error::ExecutionErr(format!( + "{name} produced more than {} MB of output. Narrow the \ + selection, or query the relation from a SQL script.", + max_stdout_bytes / 1024 / 1024 + ))); + } + truncated = true; + out_open = false; + continue; } stdout.extend_from_slice(&out_buf[..n]); } @@ -3918,7 +4057,7 @@ async fn run_capturing( .wait() .await .map_err(|e| Error::internal_err(format!("{name} failed: {e}")))?; - Ok((status, stdout, stderr)) + Ok((status, stdout, stderr, truncated)) }, ctx.worker_name, w_id, @@ -3928,14 +4067,46 @@ async fn run_capturing( })), ) .await?; - let (status, stdout, stderr) = out; - if !status.success() { + let (status, stdout, stderr, truncated) = out; + Ok(Captured { + stdout: String::from_utf8_lossy(&stdout).to_string(), + stderr: String::from_utf8_lossy(&stderr).to_string(), + // A killed child reports failure; under `Truncate` that is the ceiling's + // doing, not the project's, and the caller reads `truncated` to tell. + success: status.success(), + truncated, + }) +} + +/// `run_captured`, with a non-zero exit folded into the error — what a caller +/// that needs the command to have WORKED wants. +pub(crate) async fn run_capturing( + cmd: Command, + name: &str, + ctx: &mut JobCtx<'_>, + job_id: &Uuid, + w_id: &str, + conn: &Connection, + max_stdout_bytes: usize, +) -> error::Result { + let captured = run_captured( + cmd, + name, + ctx, + job_id, + w_id, + conn, + max_stdout_bytes, + Overflow::Fail, + ) + .await?; + if !captured.success { return Err(Error::ExecutionErr(format!( "{name} failed: {}", - String::from_utf8_lossy(&stderr) + captured.stderr ))); } - Ok(String::from_utf8_lossy(&stdout).to_string()) + Ok(captured) } /// Run a preparation command through the same child handler the build uses, so @@ -4888,7 +5059,11 @@ fn has_retryable_node(run_results: &str) -> bool { } /// Append `--vars` if the descriptor (or the run) declares any. -fn add_vars(cmd: &mut Command, descriptor: &DbtDescriptor, inv: &Invocation) -> error::Result<()> { +pub(crate) fn add_vars( + cmd: &mut Command, + descriptor: &DbtDescriptor, + inv: &Invocation, +) -> error::Result<()> { let vars = resolved_vars(descriptor, &inv.args, inv.strict)?; if !vars.is_empty() { cmd.args(["--vars", &serde_json::to_string(&vars).unwrap_or_default()]); @@ -5243,6 +5418,41 @@ fn selection_is_overridden( Ok(differs("select", &descriptor.select)? || differs("exclude", &descriptor.exclude)?) } +/// Whether this invocation rebuilds incremental models from scratch: the run +/// form's answer when it gave one, else the descriptor's — and never for a +/// `test`, which builds nothing whatever the form said. +/// +/// Shared with the column-lineage pass rather than recomputed there, because +/// `is_incremental()` branches on it: the same model compiles to different SQL — +/// a `{{ this }}` self-join, and any `ref()` inside the incremental branch — so a +/// pass that guessed would describe a build that never ran. +/// +/// `test` returns false because `dbt test` rejects `--full-refresh` outright. +/// It never arrives as a caller's `dbt_command` — the allowlist has no such +/// value — so reading only that allowlist suggests this branch is dead. It is +/// not: `run_dbt` is invoked with `"test"` directly for the `after_all` test +/// phase, and an `after_all` project with `full_refresh: true` reaches here. +pub(crate) fn full_refresh( + descriptor: &DbtDescriptor, + inv: &Invocation, + command: &str, +) -> error::Result { + if command == "test" { + return Ok(false); + } + Ok(arg_bool(&inv.args, "full_refresh")?.unwrap_or(descriptor.full_refresh)) +} + +/// Whether this run answered `full_refresh` differently from the deployed +/// descriptor. Like a selection override it changes what the graph describes, +/// since an incremental branch can carry its own `ref()`. +fn full_refresh_is_overridden( + descriptor: &DbtDescriptor, + args: &HashMap>, +) -> error::Result { + Ok(arg_bool(args, "full_refresh")?.is_some_and(|v| v != descriptor.full_refresh)) +} + /// The descriptor's named selector, unless this run named its own selection. /// /// dbt resolves `--selector` INSTEAD of `--select`, so passing both makes the @@ -6254,6 +6464,58 @@ mod tests { "an overridden selection must not publish ownership" ); } + + // `full_refresh` decides whether `is_incremental()` is true, so an + // incremental model's self-join — and any `ref()` inside that branch — + // exists in one answer and not the other. A run that flips it describes + // a different graph, and gets its own. + let mut refreshed = GraphRefresh::default(); + refreshed + .add_caller_args(&descriptor, &arg("full_refresh", "true")) + .unwrap(); + assert!(refreshed.needed()); + assert_eq!(refreshed.snapshot_job(job), Some(job)); + + // The same echo rule: the form posts the descriptor's own value back on + // every run, and reading that as an override would make each one + // caller-scoped. + let always = DbtDescriptor { full_refresh: true, ..Default::default() }; + let mut echoed_flag = GraphRefresh { profile_drift: true, ..Default::default() }; + echoed_flag + .add_caller_args(&always, &arg("full_refresh", "true")) + .unwrap(); + assert_eq!(echoed_flag.snapshot_job(job), None); + } + + /// The build and the analysis pass read this through one function, so they + /// cannot disagree about which SQL the run compiles — including for `test`, + /// which rebuilds nothing whatever the descriptor or the form said. + #[test] + fn full_refresh_is_one_answer_for_the_build_and_the_pass() { + let inv = |args: HashMap>| Invocation { + args, + raw_args: Default::default(), + envs: Default::default(), + strict: true, + deferral: None, + }; + let always = DbtDescriptor { full_refresh: true, ..Default::default() }; + let never = DbtDescriptor::default(); + let on = HashMap::from([( + "full_refresh".to_string(), + RawValue::from_string("true".to_string()).unwrap(), + )]); + + assert!(full_refresh(&always, &inv(Default::default()), "build").unwrap()); + assert!(!full_refresh(&never, &inv(Default::default()), "build").unwrap()); + assert!( + full_refresh(&never, &inv(on), "build").unwrap(), + "the form's answer wins over the descriptor's" + ); + assert!( + !full_refresh(&always, &inv(Default::default()), "test").unwrap(), + "a test builds nothing, so neither the build nor the pass may pass the flag" + ); } // `dbt retry` restores the previous run's target/ from this directory, so two diff --git a/backend/windmill-worker/src/lib.rs b/backend/windmill-worker/src/lib.rs index 8aa739ecd8..6059936af6 100644 --- a/backend/windmill-worker/src/lib.rs +++ b/backend/windmill-worker/src/lib.rs @@ -33,6 +33,7 @@ pub mod common; mod config; mod csharp_executor; +mod dbt_column_index; mod dbt_engine; mod dbt_executor; mod dbt_profiles; diff --git a/docs/dbt-runtime.md b/docs/dbt-runtime.md index acb4621b27..2414c49cc4 100644 --- a/docs/dbt-runtime.md +++ b/docs/dbt-runtime.md @@ -36,7 +36,7 @@ the dominant way dbt is orchestrated today. | 11 | Asset kind | `dbt:////` — keyed on the relation, not on dbt's node id. See below | | 12 | Graph refresh | Deploy-time, re-ingested per run only when the descriptor is dynamic, plus an explicit `parse` of the editor's buffer. See below | | 13 | Manifest storage | Sidecar table for nodes/edges; the whole manifest is kept once per environment, for deferral — see below | -| 14 | Metadata depth | Tests, strategy, tags, freshness, column descriptions. Column **lineage** is not in the manifest — see below | +| 14 | Metadata depth | Tests, strategy, tags, freshness, column descriptions. Column **lineage** and real column schemas come from the engine's parquet index, opt-in per project — see below | | 15 | Node rendering | Asset nodes per model plus one runnable node for the script | | 16 | Progress | Live, from the JSON event stream | | 17 | Test failures | Honor dbt's own `severity` | @@ -943,6 +943,8 @@ profile: select: ["tag:nightly+"] exclude: [] test_behavior: build # build | after_all | none +column_lineage: false # opt in to the static-analysis pass that + # produces column-level lineage (decision 14) vars: # typed: numbers/bools/lists keep their type, run_date: "{{ run_date }}" # and string leaves take job arguments strict: false @@ -1567,11 +1569,115 @@ answers for a machine's history rather than for the environment. So exactly one manifest is kept per (script, environment), replaced by each successful run, rather than one per version (see "Durable state per environment" above). -**Decision 14 — column lineage is not available.** The decision assumed -`manifest.json` carries column-to-column edges; it does not, in either core -engine. What it does carry is declared column *descriptions*, which are -ingested. Real column lineage would need Fusion (which does static analysis) or -a SQL-AST pass of our own, so `columnLineageGraph.ts` is not wired up for dbt. +**Decision 14 — column lineage comes from the parquet index, not the manifest.** +`manifest.json` carries no column-to-column edges, in any engine, and its +`columns` are the ones an author declared in `schema.yml`. Both halves exist in a +different artifact: `dbt compile --static-analysis strict --write-index` writes +`target/index/`, and two of its tables are `dbt.column_lineage.parquet` +(`from_node_unique_id`, `from_column_name`, `to_node_unique_id`, +`to_column_name`, `lineage_kind`) and `dbt.node_columns.parquet` (every column of +every node, with its declared type, its inferred type and its description). + +Three measured properties decide the shape of the ingest. + +**Strict analysis is a stricter dialect.** `select no_such_column from +ref(...)` is `UnresolvedIdentifier (dbt0227)` and exit 1 under `strict`, and +compiles fine under `baseline` (the default). So this is a separate `dbt compile` +with its own `--target-path`, never a flag on the build, and it is opt-in per +project: `column_lineage: true` in the descriptor. Off, nothing changes. On, a +project that cannot be analyzed keeps exactly the graph it had. + +The pass is best-effort about everything that is ITS: a wrong engine, a rejected +analysis, a missing or unreadable artifact, an over-long output and outrunning its +own time budget all degrade to partial lineage or none, plus a line in the job +log saying which. It is not best-effort about the JOB: a cancellation or the job's +own deadline fail it, because swallowing those would let a run that blew its +timeout inside an optional annotation publish a graph and report success. That +split is why the two halves have separate error contracts — the compile owns the +job's semantics and may `Err`; nothing the artifact does or fails to do is a +reason to fail a job, so an absent, unreadable or partial index is a value. The +decode still runs under the job poller, which both heartbeats through it and +ends it if the job is cancelled or completed meanwhile: the job reaching in, not +the artifact reaching out. The +budget is half the job's remaining wall clock, spent on the compile alone, so the +build that follows cannot be starved by it. + +**A failed pass still writes the index**, holding every edge of the models that +did analyze, so the artifact is read whatever the exit status and partial lineage +is a normal outcome. An unreachable *source* is milder still: `RemoteError +(dbt1014)` downgrades that model to `static_analysis: off` and the compile +succeeds. (Strict analysis queries the warehouse catalog for source schemas; a +`ref()`ed model is inferred statically and needs no built table.) + +**The flag is not the capability.** `dbt-core` 2.0.0-alpha.5 — the version +`DBT_CORE_2X_VERSION` pins — accepts `--write-index` and `--write-lineage`, and +its own `views.sql` declares views over both tables, but it writes neither +parquet; only Fusion does today. The ADAPTER decides too: an experimental one +(postgres under `DBT_ALLOW_EXPERIMENTAL_ADAPTERS`) turns static analysis off and +says so only in a warning on an otherwise successful compile. The gate is +therefore "the engine has the flag" (everything but 1.x, whose Python CLI has no +such option) plus "the file appeared", so a later release picking the feature up +needs no change here — and the job log carries the engine's own stderr whenever +no index appears, since without it "no column lineage" has no explanation. + +`lineage_kind` is stored as TEXT, not an enum. Three values exist — `copy` +(passthrough), `mod` (transformed) and `scan` (the column was read to produce the +ROW rather than the value: a join key, a `where` predicate, a `group by`) — and +the engine's own reader maps those three and passes anything else through, so the +set is the engine's to extend. All three are stored, and `copy`/`mod` are kept +first when the bound bites: a `scan` edge reaches every output column of its +model, so it is most of what a project's index holds and would draw as a complete +bipartite graph. Keeping it in the table anyway is what lets a later "show +indirect" view ask for it without every project being redeployed. + +Storage mirrors `dbt_edge` exactly: `dbt_column_edge`, keyed by (path, version, +job) with the same composite foreign key to `script`, so a version's column +lineage dies with the version and a run's snapshot with the sweep. + +**A table of its own, not `dbt_edge.column_lineage` JSONB.** Hanging the links on +the `ref()` edge they sit beneath would inherit its clone, prune, clear and +cascade paths for free, and it does not work: a model reading `{{ this }}` gets +column lineage from itself to itself, and `parent_map` has no self-loop, because +a model does not `ref()` itself. Those pairs have no `dbt_edge` row to attach to. +The loss is not hypothetical — an incremental that selects from `{{ this }}` +(`coalesce(p.dbl, s.dbl)`, `p.up as prev_up`) yields `up → prev_up` with kind +`copy`, a drawn edge meaning "this column carries the previous run's value". +Inventing self-loop `dbt_edge` rows to hold it is not an option either: that +table is `ref()` lineage. The typed column list lands in +`dbt_node.column_schema`, beside `columns` rather than merged into it — +`columns` stays what the author *declared*. + +**Stored now, served later.** This change lands the ingest and the storage; the +endpoint that draws a column trace is a follow-up. What is user-visible today is +`column_schema` — every column of a relation, typed and in the order the model +emits them — which rides the asset graph the details pane already fetches, and +replaces a panel that could only list the columns an author happened to document. +The edges sit in `dbt_column_edge` waiting for their surface. + +`column_schema` is gated on being able to read the producing project, like the +model's SQL: a column-level view is the shape of what the author wrote, one level +finer than the `ref()` graph, which is ungated only because it draws relations the +caller already sees. A share-link viewer entitled to a dbt run therefore gets its +relations and `ref()` edges, and neither the SQL nor the columns. + +**The analysis pass takes the build's own `--full-refresh`.** `is_incremental()` +branches on it, so an incremental model reading `{{ this }}` compiles its +self-join — and any `ref()` inside that branch — only when the flag is absent. A +pass that used the descriptor's default while the run overrode it would store +lineage for SQL that run never executed. For the same reason an invocation that +overrides the flag counts as `per_run_models`: its graph is its own, keyed to the +job, rather than standing as the version's. + +That flag is not the whole of it, and the rest is a property rather than a bug to +fix. `is_incremental()` is also false when the target table does not exist, so an +incremental model has **two shapes and one ingest holds one of them**: a deploy +before the first build compiles the cold shape, and the same project deployed +again once its tables exist compiles the incremental one. A static descriptor +re-ingests on neither runs nor time, so what is stored stays whatever the compile +in front of it saw. dbt has no mode that emits both, and re-analyzing per run +would buy a second `dbt compile` on every build to keep a graph nobody asked to +refresh. The contract is therefore the honest one: a version's graph describes +the compile that produced it, and a run that re-ingests describes its own run. ## Concept mapping @@ -1583,7 +1689,9 @@ a SQL-AST pass of our own, so `columnLineageGraph.ts` is not wired up for dbt. | `materialized: incremental` | `append` or `merge` (by `unique_key`) | same | | `{% snapshot %}` | `scd2` | same, incl. `_current` handling | | `unique`/`not_null`/`accepted_values`/`relationships` | `data_tests` | exact 1:1 with the four `// data_test` kinds | -| declared column metadata | `columns` on the asset node | descriptions only; see the note below | +| declared column metadata | `columns` on the asset node | descriptions only, from the manifest | +| analyzed column schema | `column_schema` on the asset node | `dbt.node_columns.parquet`, opt-in | +| column-to-column lineage | `dbt_column_edge` rows (no view yet) | `dbt.column_lineage.parquet`, opt-in | | model `tags` | node badge | `tag` | | source freshness | `freshness` | `last_success_at` chip | | `run_results.json` | materialization records | `record_materialization` | diff --git a/frontend/src/lib/components/assets/AssetGraph/types.ts b/frontend/src/lib/components/assets/AssetGraph/types.ts index 4ada1feec3..023f71e218 100644 --- a/frontend/src/lib/components/assets/AssetGraph/types.ts +++ b/frontend/src/lib/components/assets/AssetGraph/types.ts @@ -36,9 +36,14 @@ export interface DbtAssetProvenance { tags?: string[] description?: string data_tests?: DbtDataTest[] - /** Declared column metadata (name -> description). NOT column lineage: - * `manifest.json` carries none (docs/dbt-runtime.md, decision 14). */ + /** Declared column metadata (name -> description): what `manifest.json` + * carries, which is only the columns an author wrote down. */ columns?: Record + /** Every column of the relation, typed and in the order the model produces + * them, from the engine's static analysis. Present only for a project that + * opted into it (`column_lineage: true`); `manifest.json` has no such + * thing. Lockstep with Rust `DbtAssetProvenance.column_schema`. */ + column_schema?: { name: string; type?: string }[] /** A source's declared freshness policy. */ freshness?: unknown /** The model's SQL as written — the transform behind the node. Read-only: diff --git a/frontend/src/lib/components/dbt/DbtModelDetails.svelte b/frontend/src/lib/components/dbt/DbtModelDetails.svelte index 2d25dbdc00..5067902c4c 100644 --- a/frontend/src/lib/components/dbt/DbtModelDetails.svelte +++ b/frontend/src/lib/components/dbt/DbtModelDetails.svelte @@ -111,7 +111,24 @@ return typeof v === 'object' ? JSON.stringify(v) : String(v) } - let columns = $derived(Object.entries(dbt.columns ?? {})) + // The real columns where the analysis pass produced them — typed and in the + // order the model emits them — and the declared ones otherwise. The + // description comes from `columns` either way: that is the only place an + // author's prose lives, and a project documents a handful of forty. + let columns = $derived( + dbt.column_schema?.length + ? dbt.column_schema.map((c) => ({ + name: c.name, + type: c.type, + description: dbt.columns?.[c.name] ?? '' + })) + : Object.entries(dbt.columns ?? {}).map(([name, description]) => ({ + name, + type: undefined, + description + })) + ) + let columnsAreAnalyzed = $derived(!!dbt.column_schema?.length) // `dbt show` SELECTs from the node's own relation and the worker intersects // the selector with `resource_type:model`, so offering it on a seed, snapshot // or source only ever produces a failed job. @@ -202,15 +219,15 @@ {#if stalePlaceholders}
- The run arguments have changed since this graph was parsed, so these rows need not - describe the models on screen — arguments reach schemas, aliases and which models exist - at all. Refresh the models to draw and preview them under the current ones. + The run arguments have changed since this graph was parsed, so these rows need not describe + the models on screen — arguments reach schemas, aliases and which models exist at all. Refresh + the models to draw and preview them under the current ones.
{:else if staleVars}
- The run form's vars have changed since this graph was parsed. Rows are previewed under - the vars it was parsed with, so they still describe the models on screen — refresh the - models to draw and preview them under the current ones. + The run form's vars have changed since this graph was parsed. Rows are previewed under the + vars it was parsed with, so they still describe the models on screen — refresh the models to + draw and preview them under the current ones.
{/if} @@ -238,20 +255,29 @@
{#if columns.length > 0}
-
columns declared
+
+ {columnsAreAnalyzed ? 'columns' : 'columns declared'} +
- {#each columns as [name, desc] (name)} + {#each columns as col (col.name)}
- {name} - {desc} + {col.name} + {#if col.type} + {col.type} + {/if} + {col.description}
{/each}
- -
- Declared metadata — dbt reports no column-level lineage. -
+ + {#if !columnsAreAnalyzed} +
+ Declared metadata. Set `column_lineage: true` in the descriptor for the real + column schema, typed and in the order the model produces it. +
+ {/if}
{/if} {#if (dbt.data_tests?.length ?? 0) > 0} diff --git a/frontend/src/lib/script_helpers.ts b/frontend/src/lib/script_helpers.ts index c253bc64e7..b028e9360d 100644 --- a/frontend/src/lib/script_helpers.ts +++ b/frontend/src/lib/script_helpers.ts @@ -1386,6 +1386,12 @@ defer: false # resolved to that Windmill variable, so secrets stay out of this file. # env: # DBT_PASSWORD: $var:u/user/my_warehouse_password +# Real column schemas — every column typed and in the order the model produces +# it — from the engine's static analysis, which also records column-level +# lineage for a later view. Opt-in because it runs a separate dbt compile under +# --static-analysis strict, which rejects SQL the default accepts; a project it +# cannot analyze keeps the graph it has. Needs an engine that computes it. +# column_lineage: true ` // for related places search: ADD_NEW_LANG export const INITIAL_CODE = { From 33f9828c3ed15fe63fccedc1550584f15c0490ab Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 8 Sep 2026 08:29:36 +0000 Subject: [PATCH 019/144] feat: draw a dbt column trace, across projects and the pipeline boundary (#11014) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Serves the edges `dbt_column_edge` has been storing. `assets/column_lineage` answers the connected component a set of relations' columns sit in, and the details pane draws it beside the model's SQL — in the dbt editor, on the pipeline page, and for a run through `jobs/dbt_column_lineage/{id}`. The unpinned component crosses projects. A relation one project produces is another's source, so resolving owners once — for the relations asked about — stops the trace at the first boundary. Owners are resolved to a fixpoint instead, and the caller's gate is re-applied to every project the expansion discovers: reaching a relation says nothing about who may read the project on the far side of it. A pinned answer needs none of it, by version or by job: the pin says which stored graph is on screen, and another project's live graph is not part of it. One request per selection, whatever it reaches: the endpoint takes every relation at once and answers their union, so nothing is held between selections and there is no staleness, retry bookkeeping or per-click dedup to balance. The answer is bounded. A synthetic 3000-model project whose models share a column has 58k direct edges and returns 7.3MB, which no column diagram can draw; the walk is breadth-first from the asked-for relations and stops at 5000 edges, so what survives is the part nearest the selection, and `truncated` says the trace was cut rather than ended. Also adds the columns section the pipeline page's asset pane was missing, so `column_schema` is visible there and not only in the dbt editor. Claude-Session: https://claude.ai/code/session_01NY4kuFy2jAnGEzaCc1CseL Co-authored-by: Claude Opus 5 (1M context) --- ...c718161d95f7ad65a26525c5f43241267608d.json | 17 + ...fc8ee144b14d564a3e589a60678b3e68effbf.json | 17 + ...4ff3bc1143dab285e51a9b54b2c2f240b9a8c.json | 16 + ...115b2241215d660d25df7615c8b12d704e92c.json | 20 + ...ac8ee7e664e64a51972327875b1138f80db5d.json | 50 ++ ...c77cdae5377120441229490e468d47112819d.json | 15 + ...16bd03da56192754610d9b34d69e567bd1f10.json | 18 + ...3c2d1acc3cd857e089e8c6972fcd276094837.json | 16 + ...eef4f68cc3ecaf680c55a94175f6dfc2e2912.json | 16 + ...10ba39025d9c9787a71a9456241cf8a9b9d82.json | 16 + ...8ca41990e4c5d49b5baaee41acfe409bb1c24.json | 17 + ...b8b93b4a8383d8cf835a7d350056565669e7f.json | 16 + ...5989fae585115a24f19946a4d58257c93bbe2.json | 41 ++ ...ffe8aaf6dfe664c330ac8cea51a157a4fe52e.json | 17 + backend/windmill-api-assets/src/lib.rs | 442 +++++++++++++++- .../tests/dbt_pinned_graph.rs | 493 +++++++++++++++++- backend/windmill-api/openapi.yaml | 139 +++++ backend/windmill-api/src/jobs.rs | 87 +++- docs/dbt-runtime.md | 73 ++- .../AssetGraph/AssetGraphDetailsPane.svelte | 73 ++- .../AssetGraph/ColumnTraceSection.svelte | 50 ++ .../assets/AssetGraph/DbtColumnList.svelte | 50 ++ .../AssetGraph/PipelineGraphEditor.svelte | 15 +- .../AssetGraph/columnLineageGraph.test.ts | 76 +++ .../assets/AssetGraph/columnLineageGraph.ts | 75 ++- .../AssetGraph/dbtColumnLineage.svelte.ts | 120 +++++ .../src/lib/components/dbt/DbtEditor.svelte | 38 +- .../lib/components/dbt/DbtModelDetails.svelte | 70 +-- .../lib/components/dbt/DbtModelGraph.svelte | 48 +- .../(logged)/pipeline/[folder]/+page.svelte | 56 +- 30 files changed, 2072 insertions(+), 125 deletions(-) create mode 100644 backend/.sqlx/query-06c1a79bfc24b17acbd79411295c718161d95f7ad65a26525c5f43241267608d.json create mode 100644 backend/.sqlx/query-1008ed150f30b56baf17b3c6e6bfc8ee144b14d564a3e589a60678b3e68effbf.json create mode 100644 backend/.sqlx/query-210441eb7bee09afd27a927e0e44ff3bc1143dab285e51a9b54b2c2f240b9a8c.json create mode 100644 backend/.sqlx/query-23151591ed03ea6d2b017e66f54115b2241215d660d25df7615c8b12d704e92c.json create mode 100644 backend/.sqlx/query-55af8c19888ddc222a0ef2db04fac8ee7e664e64a51972327875b1138f80db5d.json create mode 100644 backend/.sqlx/query-58231bdfb04fe73a8a41601fcc3c77cdae5377120441229490e468d47112819d.json create mode 100644 backend/.sqlx/query-5cbff3d68b684f794bf43ea981716bd03da56192754610d9b34d69e567bd1f10.json create mode 100644 backend/.sqlx/query-7334351af91382d5619ff437a253c2d1acc3cd857e089e8c6972fcd276094837.json create mode 100644 backend/.sqlx/query-895a4feb0b3cc01ad711ce0eff8eef4f68cc3ecaf680c55a94175f6dfc2e2912.json create mode 100644 backend/.sqlx/query-9f0979110f86dffc7452d80ea0410ba39025d9c9787a71a9456241cf8a9b9d82.json create mode 100644 backend/.sqlx/query-b9c200365ea426ebe01b2e67c378ca41990e4c5d49b5baaee41acfe409bb1c24.json create mode 100644 backend/.sqlx/query-c793b147014cb1b6e138aed1c2eb8b93b4a8383d8cf835a7d350056565669e7f.json create mode 100644 backend/.sqlx/query-eb0df8f3f1d66dd7dfc9aedc7945989fae585115a24f19946a4d58257c93bbe2.json create mode 100644 backend/.sqlx/query-f6de1512fa3e46883b32d56fd19ffe8aaf6dfe664c330ac8cea51a157a4fe52e.json create mode 100644 frontend/src/lib/components/assets/AssetGraph/ColumnTraceSection.svelte create mode 100644 frontend/src/lib/components/assets/AssetGraph/DbtColumnList.svelte create mode 100644 frontend/src/lib/components/assets/AssetGraph/dbtColumnLineage.svelte.ts diff --git a/backend/.sqlx/query-06c1a79bfc24b17acbd79411295c718161d95f7ad65a26525c5f43241267608d.json b/backend/.sqlx/query-06c1a79bfc24b17acbd79411295c718161d95f7ad65a26525c5f43241267608d.json new file mode 100644 index 0000000000..db9d674fcf --- /dev/null +++ b/backend/.sqlx/query-06c1a79bfc24b17acbd79411295c718161d95f7ad65a26525c5f43241267608d.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id,\n resource_type, name, asset_path, tags)\n VALUES ($1, $2, $3, $4, 'model.p.stock', 'model', 'stock',\n 'u/a/wh/analytics/stock', '{}'),\n ($1, $2, $3, $4, 'model.p.stock_daily', 'model', 'stock_daily',\n 'u/a/wh/analytics/stock_daily', '{}')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "06c1a79bfc24b17acbd79411295c718161d95f7ad65a26525c5f43241267608d" +} diff --git a/backend/.sqlx/query-1008ed150f30b56baf17b3c6e6bfc8ee144b14d564a3e589a60678b3e68effbf.json b/backend/.sqlx/query-1008ed150f30b56baf17b3c6e6bfc8ee144b14d564a3e589a60678b3e68effbf.json new file mode 100644 index 0000000000..00b839de6a --- /dev/null +++ b/backend/.sqlx/query-1008ed150f30b56baf17b3c6e6bfc8ee144b14d564a3e589a60678b3e68effbf.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id,\n parent_unique_id, parent_column, child_unique_id,\n child_column, lineage_kind)\n VALUES ($1, $2, $3, $4, 'model.p.stock', 'sku', 'model.p.stock_daily', 'sku', 'copy')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "1008ed150f30b56baf17b3c6e6bfc8ee144b14d564a3e589a60678b3e68effbf" +} diff --git a/backend/.sqlx/query-210441eb7bee09afd27a927e0e44ff3bc1143dab285e51a9b54b2c2f240b9a8c.json b/backend/.sqlx/query-210441eb7bee09afd27a927e0e44ff3bc1143dab285e51a9b54b2c2f240b9a8c.json new file mode 100644 index 0000000000..bdb54cfbb9 --- /dev/null +++ b/backend/.sqlx/query-210441eb7bee09afd27a927e0e44ff3bc1143dab285e51a9b54b2c2f240b9a8c.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id,\n parent_unique_id, parent_column, child_unique_id,\n child_column, lineage_kind)\n VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000',\n 'model.p.raw_orders', 'id', 'model.p.orders', 'order_id', 'copy'),\n ($1, $2, $3, '00000000-0000-0000-0000-000000000000',\n 'model.p.raw_orders', 'status', 'model.p.orders', 'order_id', 'scan')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "210441eb7bee09afd27a927e0e44ff3bc1143dab285e51a9b54b2c2f240b9a8c" +} diff --git a/backend/.sqlx/query-23151591ed03ea6d2b017e66f54115b2241215d660d25df7615c8b12d704e92c.json b/backend/.sqlx/query-23151591ed03ea6d2b017e66f54115b2241215d660d25df7615c8b12d704e92c.json new file mode 100644 index 0000000000..50755132fc --- /dev/null +++ b/backend/.sqlx/query-23151591ed03ea6d2b017e66f54115b2241215d660d25df7615c8b12d704e92c.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id,\n resource_type, name, asset_path, tags)\n VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000', 'source.q.' || $4,\n 'source', $4, $5, '{}'),\n ($1, $2, $3, '00000000-0000-0000-0000-000000000000', 'model.q.' || $6,\n 'model', $6, $7, '{}')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "23151591ed03ea6d2b017e66f54115b2241215d660d25df7615c8b12d704e92c" +} diff --git a/backend/.sqlx/query-55af8c19888ddc222a0ef2db04fac8ee7e664e64a51972327875b1138f80db5d.json b/backend/.sqlx/query-55af8c19888ddc222a0ef2db04fac8ee7e664e64a51972327875b1138f80db5d.json new file mode 100644 index 0000000000..21759e2fe5 --- /dev/null +++ b/backend/.sqlx/query-55af8c19888ddc222a0ef2db04fac8ee7e664e64a51972327875b1138f80db5d.json @@ -0,0 +1,50 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT p.asset_path AS \"from_path!\", e.parent_column AS \"from_column!\",\n c.asset_path AS \"to_path!\", e.child_column AS \"to_column!\",\n e.lineage_kind AS \"kind!\"\n FROM unnest($2::text[], $3::bigint[], $4::uuid[])\n AS o(script_path, script_hash, job_id)\n JOIN dbt_column_edge e ON e.workspace_id = $1\n AND e.script_path = o.script_path\n AND e.job_id = o.job_id\n -- `=` still, with the NULL-to-NULL case\n -- spelled out and gated on the pin: a\n -- version-less row's hash is NULL on both\n -- sides, which `=` never matches, but\n -- `IS NOT DISTINCT FROM` would cost the\n -- equality its index bound everywhere else.\n AND (e.script_hash = o.script_hash\n OR ($5::text IS NOT NULL\n AND o.script_hash IS NULL\n AND e.script_hash IS NULL))\n JOIN dbt_node p ON p.workspace_id = e.workspace_id\n AND p.script_path = e.script_path\n AND p.script_hash IS NOT DISTINCT FROM e.script_hash\n AND p.job_id = e.job_id\n AND p.unique_id = e.parent_unique_id\n JOIN dbt_node c ON c.workspace_id = e.workspace_id\n AND c.script_path = e.script_path\n AND c.script_hash IS NOT DISTINCT FROM e.script_hash\n AND c.job_id = e.job_id\n AND c.unique_id = e.child_unique_id\n WHERE e.lineage_kind IN ('copy', 'mod')\n AND p.asset_path IS NOT NULL AND c.asset_path IS NOT NULL", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "from_path!", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "from_column!", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "to_path!", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "to_column!", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "kind!", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "TextArray", + "Int8Array", + "UuidArray", + "Text" + ] + }, + "nullable": [ + true, + false, + true, + false, + false + ] + }, + "hash": "55af8c19888ddc222a0ef2db04fac8ee7e664e64a51972327875b1138f80db5d" +} diff --git a/backend/.sqlx/query-58231bdfb04fe73a8a41601fcc3c77cdae5377120441229490e468d47112819d.json b/backend/.sqlx/query-58231bdfb04fe73a8a41601fcc3c77cdae5377120441229490e468d47112819d.json new file mode 100644 index 0000000000..822695a6c9 --- /dev/null +++ b/backend/.sqlx/query-58231bdfb04fe73a8a41601fcc3c77cdae5377120441229490e468d47112819d.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms)\n VALUES ($1, $2, $2, '{}', '{}')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "58231bdfb04fe73a8a41601fcc3c77cdae5377120441229490e468d47112819d" +} diff --git a/backend/.sqlx/query-5cbff3d68b684f794bf43ea981716bd03da56192754610d9b34d69e567bd1f10.json b/backend/.sqlx/query-5cbff3d68b684f794bf43ea981716bd03da56192754610d9b34d69e567bd1f10.json new file mode 100644 index 0000000000..8b04c9e258 --- /dev/null +++ b/backend/.sqlx/query-5cbff3d68b684f794bf43ea981716bd03da56192754610d9b34d69e567bd1f10.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id,\n parent_unique_id, parent_column, child_unique_id,\n child_column, lineage_kind)\n VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000',\n 'source.q.' || $4, 'k', 'model.q.' || $5, 'k', 'copy')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "5cbff3d68b684f794bf43ea981716bd03da56192754610d9b34d69e567bd1f10" +} diff --git a/backend/.sqlx/query-7334351af91382d5619ff437a253c2d1acc3cd857e089e8c6972fcd276094837.json b/backend/.sqlx/query-7334351af91382d5619ff437a253c2d1acc3cd857e089e8c6972fcd276094837.json new file mode 100644 index 0000000000..3a67641732 --- /dev/null +++ b/backend/.sqlx/query-7334351af91382d5619ff437a253c2d1acc3cd857e089e8c6972fcd276094837.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id,\n parent_unique_id, parent_column, child_unique_id,\n child_column, lineage_kind)\n SELECT $1, $2, $3, '00000000-0000-0000-0000-000000000000',\n 'model.p.raw_orders', 'c' || i, 'model.p.orders', 'c' || i, 'copy'\n FROM generate_series(1, 6000) i", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "7334351af91382d5619ff437a253c2d1acc3cd857e089e8c6972fcd276094837" +} diff --git a/backend/.sqlx/query-895a4feb0b3cc01ad711ce0eff8eef4f68cc3ecaf680c55a94175f6dfc2e2912.json b/backend/.sqlx/query-895a4feb0b3cc01ad711ce0eff8eef4f68cc3ecaf680c55a94175f6dfc2e2912.json new file mode 100644 index 0000000000..a498ecccd0 --- /dev/null +++ b/backend/.sqlx/query-895a4feb0b3cc01ad711ce0eff8eef4f68cc3ecaf680c55a94175f6dfc2e2912.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id,\n parent_unique_id, parent_column, child_unique_id,\n child_column, lineage_kind)\n VALUES ($1, $2, NULL, $3, 'model.p.draft_src', 'raw', 'model.p.draft', 'clean', 'mod')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "895a4feb0b3cc01ad711ce0eff8eef4f68cc3ecaf680c55a94175f6dfc2e2912" +} diff --git a/backend/.sqlx/query-9f0979110f86dffc7452d80ea0410ba39025d9c9787a71a9456241cf8a9b9d82.json b/backend/.sqlx/query-9f0979110f86dffc7452d80ea0410ba39025d9c9787a71a9456241cf8a9b9d82.json new file mode 100644 index 0000000000..49d01c2b8b --- /dev/null +++ b/backend/.sqlx/query-9f0979110f86dffc7452d80ea0410ba39025d9c9787a71a9456241cf8a9b9d82.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id,\n resource_type, name, asset_path, tags)\n VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000', 'model.p.raw_orders',\n 'model', 'raw_orders', 'u/a/wh/analytics/raw_orders', '{}'),\n ($1, $2, $3, '00000000-0000-0000-0000-000000000000', 'model.p.orders',\n 'model', 'orders', 'u/a/wh/analytics/orders', '{}')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "9f0979110f86dffc7452d80ea0410ba39025d9c9787a71a9456241cf8a9b9d82" +} diff --git a/backend/.sqlx/query-b9c200365ea426ebe01b2e67c378ca41990e4c5d49b5baaee41acfe409bb1c24.json b/backend/.sqlx/query-b9c200365ea426ebe01b2e67c378ca41990e4c5d49b5baaee41acfe409bb1c24.json new file mode 100644 index 0000000000..934742008f --- /dev/null +++ b/backend/.sqlx/query-b9c200365ea426ebe01b2e67c378ca41990e4c5d49b5baaee41acfe409bb1c24.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id,\n resource_type, name, asset_path, tags)\n VALUES ($1, $2, $3, $4, 'model.p.raw_orders', 'model', 'raw_orders',\n 'u/a/wh/analytics/raw_orders', '{}')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "b9c200365ea426ebe01b2e67c378ca41990e4c5d49b5baaee41acfe409bb1c24" +} diff --git a/backend/.sqlx/query-c793b147014cb1b6e138aed1c2eb8b93b4a8383d8cf835a7d350056565669e7f.json b/backend/.sqlx/query-c793b147014cb1b6e138aed1c2eb8b93b4a8383d8cf835a7d350056565669e7f.json new file mode 100644 index 0000000000..fee6801a6a --- /dev/null +++ b/backend/.sqlx/query-c793b147014cb1b6e138aed1c2eb8b93b4a8383d8cf835a7d350056565669e7f.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id,\n resource_type, name, asset_path, raw_code, tags)\n VALUES ($1, $2, NULL, $3, 'model.p.draft', 'model', 'draft',\n 'u/a/wh/analytics/draft', 'select 3', '{}'),\n ($1, $2, NULL, $3, 'model.p.draft_src', 'model', 'draft_src',\n 'u/a/wh/analytics/draft_src', 'select 4', '{}')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "c793b147014cb1b6e138aed1c2eb8b93b4a8383d8cf835a7d350056565669e7f" +} diff --git a/backend/.sqlx/query-eb0df8f3f1d66dd7dfc9aedc7945989fae585115a24f19946a4d58257c93bbe2.json b/backend/.sqlx/query-eb0df8f3f1d66dd7dfc9aedc7945989fae585115a24f19946a4d58257c93bbe2.json new file mode 100644 index 0000000000..854ed61857 --- /dev/null +++ b/backend/.sqlx/query-eb0df8f3f1d66dd7dfc9aedc7945989fae585115a24f19946a4d58257c93bbe2.json @@ -0,0 +1,41 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT DISTINCT n.script_path AS \"script_path!\", n.script_hash, n.job_id AS \"job_id!\"\n FROM dbt_node n\n WHERE n.workspace_id = $1 AND n.asset_path = ANY($2)\n -- The run's snapshot, or the deployed graph when that job stored\n -- none -- a build pins only if it wrote one.\n AND n.job_id = CASE WHEN $5::uuid IS NOT NULL AND EXISTS (\n SELECT 1 FROM dbt_graph_snapshot g\n WHERE g.workspace_id = $1 AND g.job_id = $5)\n THEN $5::uuid\n ELSE '00000000-0000-0000-0000-000000000000'::uuid END\n -- The gate, re-decided for every project the walk reaches. That\n -- is what resolving owners in a loop is for: being entitled to\n -- one project is not being entitled to the one that declares a\n -- relation it hands over.\n AND ( $6\n OR n.script_path = ANY($7)\n OR EXISTS ( SELECT 1 FROM unnest($8::text[]) AS pfx\n WHERE n.script_path = pfx\n OR left(n.script_path, length(pfx) + 1) = pfx || '/' ) )\n AND CASE\n -- Pinned: which version comes from a job this caller was\n -- already granted, so `script` does not decide THAT -- but\n -- it still decides whether the project may be read, the\n -- same second gate `script_visible` is on the graph. Being\n -- entitled to a run is not being entitled to the SQL\n -- behind it, and column lineage is that SQL's shape. A\n -- version-less row is exempt because it is an editor\n -- buffer, which has no `script` row to ask and reaches\n -- this only through the parse job that wrote it.\n --\n -- One project answers, so a pinned trace never crosses\n -- into another: neither does the graph it annotates.\n WHEN $4::text IS NOT NULL\n THEN n.script_path = $4 AND n.script_hash IS NOT DISTINCT FROM $3::bigint\n AND ($3::bigint IS NULL OR EXISTS (\n SELECT 1 FROM script sc\n WHERE sc.workspace_id = $1 AND sc.path = n.script_path\n AND sc.hash = $3))\n -- A named version: the deployed one an editor is drawing.\n -- `script` is read under RLS, so this is the visibility\n -- check as well as the existence one. A hash names one\n -- script row, so this arm answers for one project too —\n -- and deliberately: a pin says which stored graph is on\n -- screen, and another project's live graph is not it.\n WHEN $3::bigint IS NOT NULL\n THEN n.script_hash = $3 AND EXISTS (\n SELECT 1 FROM script sc\n WHERE sc.workspace_id = $1 AND sc.path = n.script_path\n AND sc.hash = $3)\n -- Otherwise the version deployed now: an older one's rows\n -- outlive it in `dbt_node` until the sweep, and describe a\n -- project that is no longer what runs. `language` narrows\n -- it the way the graph's own resolution does, so a path\n -- that has since become a script of another kind draws and\n -- explains the same version rather than disagreeing. Read\n -- under RLS, so a project the caller cannot see resolves\n -- to NULL and matches nothing.\n ELSE n.script_hash = (\n SELECT sc.hash FROM script sc\n WHERE sc.workspace_id = $1 AND sc.path = n.script_path\n AND sc.language = 'dbt'\n AND sc.deleted = false AND sc.archived = false\n ORDER BY sc.created_at DESC LIMIT 1)\n END", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "script_path!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "script_hash", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "job_id!", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Text", + "TextArray", + "Int8", + "Text", + "Uuid", + "Bool", + "TextArray", + "TextArray" + ] + }, + "nullable": [ + false, + true, + false + ] + }, + "hash": "eb0df8f3f1d66dd7dfc9aedc7945989fae585115a24f19946a4d58257c93bbe2" +} diff --git a/backend/.sqlx/query-f6de1512fa3e46883b32d56fd19ffe8aaf6dfe664c330ac8cea51a157a4fe52e.json b/backend/.sqlx/query-f6de1512fa3e46883b32d56fd19ffe8aaf6dfe664c330ac8cea51a157a4fe52e.json new file mode 100644 index 0000000000..ccb9863853 --- /dev/null +++ b/backend/.sqlx/query-f6de1512fa3e46883b32d56fd19ffe8aaf6dfe664c330ac8cea51a157a4fe52e.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id,\n parent_unique_id, parent_column, child_unique_id,\n child_column, lineage_kind)\n VALUES ($1, $2, $3, $4, 'model.p.raw_orders', 'id', 'model.p.orders', 'order_id',\n 'copy')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "f6de1512fa3e46883b32d56fd19ffe8aaf6dfe664c330ac8cea51a157a4fe52e" +} diff --git a/backend/windmill-api-assets/src/lib.rs b/backend/windmill-api-assets/src/lib.rs index 51bfa6c2c7..4b86013e7d 100644 --- a/backend/windmill-api-assets/src/lib.rs +++ b/backend/windmill-api-assets/src/lib.rs @@ -6,6 +6,7 @@ use axum::{ use serde::{Deserialize, Serialize}; use serde_json::Value; use sqlx::Row; +use std::collections::{BTreeSet, HashMap, HashSet, VecDeque}; use windmill_common::{ assets::{parse_asset_trigger_ref, AssetKind, AssetUsageKind}, db::UserDB, @@ -13,7 +14,9 @@ use windmill_common::{ utils::escape_ilike_pattern, }; -use windmill_api_auth::{build_scope_path_predicate, ApiAuthed}; +use windmill_api_auth::{ + build_scope_path_filter, build_scope_path_predicate, ApiAuthed, ScopePathFilter, +}; // Partition-range backfill preview. The logic (producer resolution, range // enumeration, status join) is enterprise: the `private` build compiles the @@ -33,6 +36,7 @@ pub fn workspaced_service() -> Router { .route("/list_by_usages", post(list_assets_by_usages)) .route("/list_favorites", get(list_favorites)) .route("/graph", get(asset_graph)) + .route("/column_lineage", get(dbt_column_lineage)) .route("/pipelines", get(list_pipeline_folders)) .route("/partitions", get(list_partitions)) .route("/partitions_in_range", get(list_partitions_in_range)) @@ -957,6 +961,442 @@ struct DbtLineageEdge { to_asset_path: String, } +/// One column-to-column edge, in the same terms: the two relations and the two +/// columns, never dbt's node ids. +#[derive(Serialize, Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +struct DbtColumnLineageEdge { + from_asset_path: String, + from_column: String, + to_asset_path: String, + to_column: String, + /// dbt's own word for how the value travelled: `copy` (passthrough), `mod` + /// (transformed), `scan` (read to produce the ROW rather than the value — a + /// join key, a predicate, a `group by`). Sent verbatim, including a kind + /// this engine version invented, because the renderer decides what a kind + /// means and the set is the engine's. + kind: String, +} + +/// The dbt relations a view is tracing, and which stored graph to read them +/// from. +/// +/// Several relations, answered as one union, because ONE selection reaches +/// several: a script's output column can be derived from columns of several dbt +/// models, and a model's columns can be consumed by scripts that feed others. +/// Asking per relation instead is a request per boundary plus the bookkeeping to +/// stitch the answers together and decide which of them is still current — which +/// is a cache, and is what taking them together exists to not need. +pub struct ColumnLineageQuery { + /// The `dbt://` relations whose lineage to return. + pub asset_paths: Vec, + /// The deployed version a view is drawing, when it is drawing one — the dbt + /// editor, which shows a single project as of a single deploy. + /// + /// A version-pinned answer is that version's project ALONE, the same as a + /// job-pinned one: the pin exists so the trace describes the stored graph on + /// screen, and another project's live graph is not part of it. Only the + /// unpinned answer crosses projects. + /// + /// A run's or an editor buffer's graph is NOT reachable from here: it pins + /// to a job, and that costs the job-read gate. + pub dbt_script_hash: Option, +} + +impl ColumnLineageQuery { + /// Built from the raw pairs rather than deserialized as a struct, because + /// `asset_path` REPEATS and `serde_urlencoded` — what `Query` deserializes + /// with — reads no sequence from a repeated key. A GET rather than a POST + /// body carrying the list: the method decides a scoped token's action, so a + /// POST would ask `assets:write` for a read and refuse a read-only token + /// outright. + pub fn from_query_pairs(pairs: Vec<(String, String)>) -> windmill_common::error::Result { + let mut asset_paths: Vec = Vec::new(); + let mut dbt_script_hash = None; + for (key, value) in pairs { + match key.as_str() { + "asset_path" => asset_paths.push(value), + // Hex, like every other script-hash parameter, so a page can + // pass `job.script_hash` verbatim. + "dbt_script_hash" => { + dbt_script_hash = + Some(serde_json::from_value(Value::String(value)).map_err(|_| { + windmill_common::error::Error::BadRequest( + "dbt_script_hash is not a script hash".to_string(), + ) + })?) + } + _ => {} + } + } + // REFUSED, not answered empty. A caller that named no relation — or + // misspelled the parameter — asked for something, and an empty component + // is what a relation with no lineage returns, so answering that way says + // "this has none" for a question that was never asked. + if asset_paths.is_empty() { + return Err(windmill_common::error::Error::BadRequest( + "at least one asset_path is required".to_string(), + )); + } + Ok(ColumnLineageQuery { asset_paths, dbt_script_hash }) + } +} + +#[derive(Serialize, Debug, PartialEq, Eq)] +pub struct ColumnLineageResponse { + /// Direct (`copy` / `mod`) column edges of the component the asked-for + /// relations' columns sit in, in the terms the canvas draws. Empty when no + /// project involved asked for the analysis pass, which is the ordinary case. + edges: Vec, + /// The component reaches further than what is here: `edges` holds the part + /// nearest the asked-for relations. Said rather than silently cut, because a + /// trace that stops short is otherwise indistinguishable from one that ends. + truncated: bool, +} + +/// How many edges one trace may carry back. The renderer draws a box per column, +/// so a component past this is unreadable however it is served — a synthetic +/// 3000-model project whose models share a column returns 58k direct edges and +/// 7.3MB. Applied over the walk, which is the LAST filter, so what survives is +/// the part nearest the selection rather than an arbitrary slice of it. +const MAX_TRACE_EDGES: usize = 5_000; + +/// How many times one trace may discover a project it has not read yet. Each +/// round costs a gate and a fetch, and a component crossing this many projects +/// has already outgrown what the canvas can show; stopping says what the edge +/// bound says. +const MAX_OWNER_ROUNDS: usize = 8; + +/// How many edges one trace may HOLD while walking, as opposed to answer with. +/// A project's edges arrive whole — the walk decides what is in the component, +/// so a `LIMIT` on the fetch would cut an arbitrary set that need not even +/// contain the asked-for relation — and a project is bounded at ingest by +/// `MAX_COLUMN_EDGES`, which is 200k. Rounds are what this bounds: reading a +/// second project's worth on top of an already outsized first one buys nothing, +/// since the walk is going to stop at `MAX_TRACE_EDGES` regardless. +const MAX_HELD_EDGES: usize = 100_000; + +/// A stored project graph: a deployed version, or one job's snapshot of it. +/// `script_hash` is NULL for an editor buffer's parse, which names no version. +type ProjectVersion = (String, Option, uuid::Uuid); + +async fn dbt_column_lineage( + authed: ApiAuthed, + Path(w_id): Path, + Extension(user_db): Extension, + Query(pairs): Query>, +) -> JsonResult { + // `None`: pinning to one run is job-scoped and this endpoint is authorized + // as `assets:read`. See `dbt_column_lineage_for`. + let q = ColumnLineageQuery::from_query_pairs(pairs)?; + dbt_column_lineage_for(&authed, &w_id, user_db, q, None).await +} + +/// The column-level lineage the asked-for relations sit in, optionally as one +/// run saw it. +/// +/// AUTHORIZES NOTHING BY ITSELF, on the same contract as `asset_graph_for`: +/// `assets:read` always, and the job-read gate for `Some(pinned)`, whose path +/// and hash are then taken from that job's row rather than from the caller. +/// +/// A column trace is transitive and a relation is not owned by one project, so +/// the answer grows a project at a time: resolve who owns the relations reached +/// so far, read their edges, walk, and repeat for the relations that walk newly +/// reached. Every round re-applies the caller's gate to the projects it +/// discovers — a relation being reachable from a project the caller may read +/// says nothing about the project on the other side of it. +pub async fn dbt_column_lineage_for( + authed: &ApiAuthed, + w_id: &str, + user_db: UserDB, + q: ColumnLineageQuery, + pinned: Option, +) -> JsonResult { + // A column-level view is the shape of what the author WROTE, so it takes the + // model's own gate rather than the relation's. + let (scope_all, scope_exact, scope_prefix) = + match build_scope_path_filter(authed, "scripts", "read") { + ScopePathFilter::AllowAll => (true, Vec::new(), Vec::new()), + ScopePathFilter::Restricted { exact, prefix } => (false, exact, prefix), + }; + let (pinned_path, script_hash) = match pinned.as_ref() { + // The job's own version, so a pin cannot name one project's run while + // claiming another's version — including when it names NONE, which is + // the editor buffer. + Some(p) => (Some(p.script_path.as_str()), p.script_hash), + None => (None, q.dbt_script_hash.map(|h| h.0)), + }; + let pinned_job_id = pinned.as_ref().map(|p| p.job_id); + + let seeds: BTreeSet = q.asset_paths.into_iter().collect(); + let mut tx = user_db.begin(authed).await?; + + // Relations whose owners have been asked for, project graphs already read, + // and the edges they yielded. These are what end the loop: a round asks only + // about relations not asked about before and reads only projects not read + // before, so it stops as soon as one of the two runs out. + let mut asked: HashSet = HashSet::new(); + let mut read: HashSet = HashSet::new(); + let mut edges: Vec = Vec::new(); + let mut answer: Vec = Vec::new(); + let mut pending: Vec = seeds.iter().cloned().collect(); + let mut truncated = false; + + for _ in 0..MAX_OWNER_ROUNDS { + if pending.is_empty() { + break; + } + // Which project version owns each of these relations, under this + // caller's access. Usually one row per relation; a relation a second + // project declares as a source has two, and each answers for its own + // lineage. + let owners = sqlx::query!( + r#"SELECT DISTINCT n.script_path AS "script_path!", n.script_hash, n.job_id AS "job_id!" + FROM dbt_node n + WHERE n.workspace_id = $1 AND n.asset_path = ANY($2) + -- The run's snapshot, or the deployed graph when that job stored + -- none -- a build pins only if it wrote one. + AND n.job_id = CASE WHEN $5::uuid IS NOT NULL AND EXISTS ( + SELECT 1 FROM dbt_graph_snapshot g + WHERE g.workspace_id = $1 AND g.job_id = $5) + THEN $5::uuid + ELSE '00000000-0000-0000-0000-000000000000'::uuid END + -- The gate, re-decided for every project the walk reaches. That + -- is what resolving owners in a loop is for: being entitled to + -- one project is not being entitled to the one that declares a + -- relation it hands over. + AND ( $6 + OR n.script_path = ANY($7) + OR EXISTS ( SELECT 1 FROM unnest($8::text[]) AS pfx + WHERE n.script_path = pfx + OR left(n.script_path, length(pfx) + 1) = pfx || '/' ) ) + AND CASE + -- Pinned: which version comes from a job this caller was + -- already granted, so `script` does not decide THAT -- but + -- it still decides whether the project may be read, the + -- same second gate `script_visible` is on the graph. Being + -- entitled to a run is not being entitled to the SQL + -- behind it, and column lineage is that SQL's shape. A + -- version-less row is exempt because it is an editor + -- buffer, which has no `script` row to ask and reaches + -- this only through the parse job that wrote it. + -- + -- One project answers, so a pinned trace never crosses + -- into another: neither does the graph it annotates. + WHEN $4::text IS NOT NULL + THEN n.script_path = $4 AND n.script_hash IS NOT DISTINCT FROM $3::bigint + AND ($3::bigint IS NULL OR EXISTS ( + SELECT 1 FROM script sc + WHERE sc.workspace_id = $1 AND sc.path = n.script_path + AND sc.hash = $3)) + -- A named version: the deployed one an editor is drawing. + -- `script` is read under RLS, so this is the visibility + -- check as well as the existence one. A hash names one + -- script row, so this arm answers for one project too — + -- and deliberately: a pin says which stored graph is on + -- screen, and another project's live graph is not it. + WHEN $3::bigint IS NOT NULL + THEN n.script_hash = $3 AND EXISTS ( + SELECT 1 FROM script sc + WHERE sc.workspace_id = $1 AND sc.path = n.script_path + AND sc.hash = $3) + -- Otherwise the version deployed now: an older one's rows + -- outlive it in `dbt_node` until the sweep, and describe a + -- project that is no longer what runs. `language` narrows + -- it the way the graph's own resolution does, so a path + -- that has since become a script of another kind draws and + -- explains the same version rather than disagreeing. Read + -- under RLS, so a project the caller cannot see resolves + -- to NULL and matches nothing. + ELSE n.script_hash = ( + SELECT sc.hash FROM script sc + WHERE sc.workspace_id = $1 AND sc.path = n.script_path + AND sc.language = 'dbt' + AND sc.deleted = false AND sc.archived = false + ORDER BY sc.created_at DESC LIMIT 1) + END"#, + w_id, + &pending[..], + script_hash, + pinned_path, + pinned_job_id, + scope_all, + &scope_exact[..], + &scope_prefix[..], + ) + .fetch_all(&mut *tx) + .await?; + asked.extend(pending.drain(..)); + + let fresh: Vec = owners + .into_iter() + .map(|o| (o.script_path, o.script_hash, o.job_id)) + .filter(|k| read.insert(k.clone())) + .collect(); + if fresh.is_empty() { + break; + } + let fresh_paths: Vec = fresh.iter().map(|k| k.0.clone()).collect(); + let fresh_hashes: Vec> = fresh.iter().map(|k| k.1).collect(); + let fresh_jobs: Vec = fresh.iter().map(|k| k.2).collect(); + + // DIRECT kinds only. `scan` -- the column was read to produce the ROW, + // not the value -- reaches every output column of its model, so it is + // most of a project's stored lineage and none of what a trace draws. + // It stays in the table for a later view to ask for. + let rows = sqlx::query!( + r#"SELECT p.asset_path AS "from_path!", e.parent_column AS "from_column!", + c.asset_path AS "to_path!", e.child_column AS "to_column!", + e.lineage_kind AS "kind!" + FROM unnest($2::text[], $3::bigint[], $4::uuid[]) + AS o(script_path, script_hash, job_id) + JOIN dbt_column_edge e ON e.workspace_id = $1 + AND e.script_path = o.script_path + AND e.job_id = o.job_id + -- `=` still, with the NULL-to-NULL case + -- spelled out and gated on the pin: a + -- version-less row's hash is NULL on both + -- sides, which `=` never matches, but + -- `IS NOT DISTINCT FROM` would cost the + -- equality its index bound everywhere else. + AND (e.script_hash = o.script_hash + OR ($5::text IS NOT NULL + AND o.script_hash IS NULL + AND e.script_hash IS NULL)) + JOIN dbt_node p ON p.workspace_id = e.workspace_id + AND p.script_path = e.script_path + AND p.script_hash IS NOT DISTINCT FROM e.script_hash + AND p.job_id = e.job_id + AND p.unique_id = e.parent_unique_id + JOIN dbt_node c ON c.workspace_id = e.workspace_id + AND c.script_path = e.script_path + AND c.script_hash IS NOT DISTINCT FROM e.script_hash + AND c.job_id = e.job_id + AND c.unique_id = e.child_unique_id + WHERE e.lineage_kind IN ('copy', 'mod') + AND p.asset_path IS NOT NULL AND c.asset_path IS NOT NULL"#, + w_id, + &fresh_paths[..], + &fresh_hashes[..] as &[Option], + &fresh_jobs[..], + pinned_path, + ) + .fetch_all(&mut *tx) + .await?; + + edges.extend(rows.into_iter().map(|r| DbtColumnLineageEdge { + from_asset_path: r.from_path, + from_column: r.from_column, + to_asset_path: r.to_path, + to_column: r.to_column, + kind: r.kind, + })); + // Two projects can describe one relation, so the same edge can arrive + // twice. Sorted as well as deduplicated: the walk reads the incidence + // lists in this order, so the answer does not depend on which round a + // project was discovered in. + edges.sort(); + edges.dedup(); + + let walked = component(&edges, &seeds); + answer = walked.edges; + truncated = walked.truncated; + if truncated { + break; + } + // The relations the walk newly reached. Their owners are the next round's + // question: this project declares them, and so may another. + pending = answer + .iter() + .flat_map(|e| [&e.from_asset_path, &e.to_asset_path]) + .filter(|p| !asked.contains(*p)) + .cloned() + .collect::>() + .into_iter() + .collect(); + // Stop discovering projects once the held set is outsized. The tail + // below reports what that leaves unresolved, and reports nothing when + // the walk had already reached everything. + if edges.len() >= MAX_HELD_EDGES { + break; + } + } + tx.commit().await?; + // Out of rounds with relations still unresolved: more of the component + // exists, which is what hitting the edge bound also means. + Ok(Json(ColumnLineageResponse { + truncated: truncated || !pending.is_empty(), + edges: answer, + })) +} + +struct WalkedComponent { + edges: Vec, + truncated: bool, +} + +/// The edges of the connected component the asked-for relations sit in, nearest +/// first and at most `MAX_TRACE_EDGES` of them. +/// +/// The canvas lays out the component of the selected relation's columns, so a +/// project's other model families are edges nothing it draws can reach. Walked +/// here rather than in SQL: a recursive CTE has no index to walk, so it rescans +/// the whole edge set once per level — measured at 1.24s against 59ms for the +/// query alone on a 3000-model project, for a walk that is microseconds over a +/// map. Columns are keyed by relation, not by project, which is how the canvas +/// keys them too: two projects describing one relation draw one node. +/// +/// Breadth-first, so the bound cuts the far end of the trace rather than an +/// arbitrary part of it. +fn component(edges: &[DbtColumnLineageEdge], seeds: &BTreeSet) -> WalkedComponent { + let mut incident: HashMap<(&str, &str), Vec> = HashMap::new(); + for (i, e) in edges.iter().enumerate() { + incident + .entry((&e.from_asset_path, &e.from_column)) + .or_default() + .push(i); + incident + .entry((&e.to_asset_path, &e.to_column)) + .or_default() + .push(i); + } + let mut start: Vec<(&str, &str)> = incident + .keys() + .filter(|(path, _)| seeds.contains(*path)) + .copied() + .collect(); + start.sort(); + let mut seen_node: HashSet<(&str, &str)> = start.iter().copied().collect(); + let mut queue: VecDeque<(&str, &str)> = start.into(); + let mut taken = vec![false; edges.len()]; + let mut kept: Vec = Vec::new(); + let mut truncated = false; + 'walk: while let Some(node) = queue.pop_front() { + for &i in incident.get(&node).map(Vec::as_slice).unwrap_or_default() { + if std::mem::replace(&mut taken[i], true) { + continue; + } + if kept.len() == MAX_TRACE_EDGES { + truncated = true; + break 'walk; + } + kept.push(i); + let e = &edges[i]; + let ends = [ + (e.from_asset_path.as_str(), e.from_column.as_str()), + (e.to_asset_path.as_str(), e.to_column.as_str()), + ]; + for end in ends { + if seen_node.insert(end) { + queue.push_back(end); + } + } + } + } + // Back into edge order, so a response does not carry the walk's shape. + kept.sort_unstable(); + WalkedComponent { edges: kept.into_iter().map(|i| edges[i].clone()).collect(), truncated } +} + async fn asset_graph( authed: ApiAuthed, Path(w_id): Path, diff --git a/backend/windmill-api-assets/tests/dbt_pinned_graph.rs b/backend/windmill-api-assets/tests/dbt_pinned_graph.rs index f2bb98a46c..75299f4d9f 100644 --- a/backend/windmill-api-assets/tests/dbt_pinned_graph.rs +++ b/backend/windmill-api-assets/tests/dbt_pinned_graph.rs @@ -8,7 +8,9 @@ //! whole dbt half, and every later fix in this area re-touched one of the two. use sqlx::{Pool, Postgres}; -use windmill_api_assets::{asset_graph_for, GraphQuery, PinnedRun}; +use windmill_api_assets::{ + asset_graph_for, dbt_column_lineage_for, ColumnLineageQuery, GraphQuery, PinnedRun, +}; use windmill_api_auth::ApiAuthed; use windmill_common::db::UserDB; @@ -84,6 +86,34 @@ async fn seed(db: &Pool, job: uuid::Uuid) { .execute(db) .await .unwrap(); + // The relation it reads, and the column edge between them. + sqlx::query!( + "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id, + resource_type, name, asset_path, tags) + VALUES ($1, $2, $3, $4, 'model.p.raw_orders', 'model', 'raw_orders', + 'u/a/wh/analytics/raw_orders', '{}')", + WS, + PATH, + HASH, + job + ) + .execute(db) + .await + .unwrap(); + sqlx::query!( + "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id, + parent_unique_id, parent_column, child_unique_id, + child_column, lineage_kind) + VALUES ($1, $2, $3, $4, 'model.p.raw_orders', 'id', 'model.p.orders', 'order_id', + 'copy')", + WS, + PATH, + HASH, + job + ) + .execute(db) + .await + .unwrap(); // A test node, for the arguments it carries: `accepted_values` spells out a // column's domain. sqlx::query!( @@ -365,7 +395,25 @@ async fn seed_editor_graph(db: &Pool, job: uuid::Uuid) { r#"INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id, resource_type, name, asset_path, raw_code, tags) VALUES ($1, $2, NULL, $3, 'model.p.draft', 'model', 'draft', - 'u/a/wh/analytics/draft', 'select 3', '{}')"#, + 'u/a/wh/analytics/draft', 'select 3', '{}'), + ($1, $2, NULL, $3, 'model.p.draft_src', 'model', 'draft_src', + 'u/a/wh/analytics/draft_src', 'select 4', '{}')"#, + WS, + PATH, + job + ) + .execute(db) + .await + .unwrap(); + // A version-less row's `script_hash` is NULL on both sides of every join and + // every visibility check, and `= NULL` is never true — so the column edges + // need the same NULL arm the node query has, or a buffer parse renders its + // columns and none of their lineage. + sqlx::query!( + "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id, + parent_unique_id, parent_column, child_unique_id, + child_column, lineage_kind) + VALUES ($1, $2, NULL, $3, 'model.p.draft_src', 'raw', 'model.p.draft', 'clean', 'mod')", WS, PATH, job @@ -451,3 +499,444 @@ async fn an_editor_graph_renders_only_through_its_own_job(db: Pool) { "nor of a run of the deployed version: {deployed_run}" ); } + +async fn column_lineage_q( + db: &Pool, + authed: &ApiAuthed, + pairs: Vec<(String, String)>, + pinned: Option, +) -> serde_json::Value { + let res = dbt_column_lineage_for( + authed, + WS, + UserDB::new(db.clone()), + ColumnLineageQuery::from_query_pairs(pairs).unwrap(), + pinned, + ) + .await + .unwrap(); + serde_json::to_value(&res.0).unwrap() +} + +async fn column_lineage( + db: &Pool, + authed: &ApiAuthed, + asset_paths: &[&str], + pinned: Option, +) -> serde_json::Value { + let pairs = asset_paths + .iter() + .map(|p| ("asset_path".to_string(), p.to_string())) + .collect(); + column_lineage_q(db, authed, pairs, pinned).await +} + +/// The buffer parse's own lineage, which is the case the versionless rows exist +/// for. Its `script_hash` is NULL on both sides of every join and every +/// visibility check, and `= NULL` is never true — so the versionless arm has to +/// be written for it, or a parse renders its columns and none of their lineage. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn an_editor_buffers_column_lineage_answers_through_its_job(db: Pool) { + let parse = uuid::Uuid::from_u128(9); + seed(&db, uuid::Uuid::from_u128(7)).await; + seed_editor_graph(&db, parse).await; + let admin = ApiAuthed { is_admin: true, ..outsider() }; + + let pinned = PinnedRun { job_id: parse, script_path: PATH.to_string(), script_hash: None }; + assert_eq!( + column_lineage(&db, &admin, &["u/a/wh/analytics/draft"], Some(pinned)).await["edges"], + serde_json::json!([{ + "from_asset_path": "u/a/wh/analytics/draft_src", + "from_column": "raw", + "to_asset_path": "u/a/wh/analytics/draft", + "to_column": "clean", + "kind": "mod", + }]), + ); + // Unpinned, the same relation resolves through the deployed version, which + // never heard of the buffer's models. + assert_eq!( + column_lineage(&db, &admin, &["u/a/wh/analytics/draft"], None).await["edges"], + serde_json::json!([]), + "a buffer's lineage is reachable only through the job that parsed it" + ); +} + +/// Being entitled to a RUN is not being entitled to the SQL behind it, and +/// column lineage is that SQL's shape. The pinned graph draws the relations for +/// a share-link viewer and redacts what the author wrote; the lineage is the +/// second, and resolving the version from the job must not be mistaken for +/// deciding that too. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn a_pinned_run_does_not_hand_over_the_projects_column_lineage(db: Pool) { + let job = uuid::Uuid::from_u128(7); + seed(&db, job).await; + let pinned = + || PinnedRun { job_id: job, script_path: PATH.to_string(), script_hash: Some(HASH) }; + + assert_eq!( + column_lineage( + &db, + &outsider(), + &["u/a/wh/analytics/orders"], + Some(pinned()) + ) + .await["edges"], + serde_json::json!([]), + "the run renders for them, its column-level shape does not" + ); + assert_eq!( + column_lineage( + &db, + &ApiAuthed { is_admin: true, ..outsider() }, + &["u/a/wh/analytics/orders"], + Some(pinned()) + ) + .await["edges"], + serde_json::json!([{ + "from_asset_path": "u/a/wh/analytics/raw_orders", + "from_column": "id", + "to_asset_path": "u/a/wh/analytics/orders", + "to_column": "order_id", + "kind": "copy", + }]), + "while a reader of the project gets it" + ); +} + +/// The deployed version of the same two models, plus a `scan` edge beside the +/// direct one: `seed`'s rows are a run's snapshot, and the unpinned answer is +/// the version's own graph. +async fn seed_deployed_orders(db: &Pool) { + sqlx::query!( + "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id, + resource_type, name, asset_path, tags) + VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000', 'model.p.raw_orders', + 'model', 'raw_orders', 'u/a/wh/analytics/raw_orders', '{}'), + ($1, $2, $3, '00000000-0000-0000-0000-000000000000', 'model.p.orders', + 'model', 'orders', 'u/a/wh/analytics/orders', '{}')", + WS, + PATH, + HASH, + ) + .execute(db) + .await + .unwrap(); + sqlx::query!( + "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id, + parent_unique_id, parent_column, child_unique_id, + child_column, lineage_kind) + VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000', + 'model.p.raw_orders', 'id', 'model.p.orders', 'order_id', 'copy'), + ($1, $2, $3, '00000000-0000-0000-0000-000000000000', + 'model.p.raw_orders', 'status', 'model.p.orders', 'order_id', 'scan')", + WS, + PATH, + HASH, + ) + .execute(db) + .await + .unwrap(); +} + +/// A column-level view is the shape of what the author WROTE, so it takes the +/// script's own gate — the same one that keeps `raw_code` behind access to the +/// project. `scan` says the column was read to produce the ROW rather than the +/// value, so it reaches every output column of its model and is never served. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn column_lineage_takes_the_scripts_gate_and_only_the_direct_kinds(db: Pool) { + seed(&db, uuid::Uuid::from_u128(7)).await; + seed_deployed_orders(&db).await; + + let admin = ApiAuthed { is_admin: true, ..outsider() }; + assert_eq!( + column_lineage(&db, &admin, &["u/a/wh/analytics/orders"], None).await["edges"], + serde_json::json!([{ + "from_asset_path": "u/a/wh/analytics/raw_orders", + "from_column": "id", + "to_asset_path": "u/a/wh/analytics/orders", + "to_column": "order_id", + "kind": "copy", + }]), + "the direct edge, and not the `scan` one beside it" + ); + assert_eq!( + column_lineage(&db, &outsider(), &["u/a/wh/analytics/orders"], None).await["edges"], + serde_json::json!([]), + "and nothing at all for a caller who cannot read the project" + ); +} + +/// One project routinely holds model families that share no column, and the +/// canvas lays out the connected component of the selected relation's columns. +/// Answering with the project's other components sends edges nothing can draw. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn column_lineage_stops_at_the_selected_relations_component(db: Pool) { + let job = uuid::Uuid::from_u128(7); + seed(&db, job).await; + // A second family in the same project version, reaching neither of the two + // relations `seed` wired together. + sqlx::query!( + "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id, + resource_type, name, asset_path, tags) + VALUES ($1, $2, $3, $4, 'model.p.stock', 'model', 'stock', + 'u/a/wh/analytics/stock', '{}'), + ($1, $2, $3, $4, 'model.p.stock_daily', 'model', 'stock_daily', + 'u/a/wh/analytics/stock_daily', '{}')", + WS, + PATH, + HASH, + job + ) + .execute(&db) + .await + .unwrap(); + sqlx::query!( + "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id, + parent_unique_id, parent_column, child_unique_id, + child_column, lineage_kind) + VALUES ($1, $2, $3, $4, 'model.p.stock', 'sku', 'model.p.stock_daily', 'sku', 'copy')", + WS, + PATH, + HASH, + job + ) + .execute(&db) + .await + .unwrap(); + + let admin = ApiAuthed { is_admin: true, ..outsider() }; + let pinned = + || PinnedRun { job_id: job, script_path: PATH.to_string(), script_hash: Some(HASH) }; + assert_eq!( + column_lineage(&db, &admin, &["u/a/wh/analytics/orders"], Some(pinned())).await["edges"], + serde_json::json!([{ + "from_asset_path": "u/a/wh/analytics/raw_orders", + "from_column": "id", + "to_asset_path": "u/a/wh/analytics/orders", + "to_column": "order_id", + "kind": "copy", + }]), + "the orders family, and not the stock one beside it in the same project" + ); + assert_eq!( + column_lineage( + &db, + &admin, + &["u/a/wh/analytics/stock_daily"], + Some(pinned()) + ) + .await["edges"], + serde_json::json!([{ + "from_asset_path": "u/a/wh/analytics/stock", + "from_column": "sku", + "to_asset_path": "u/a/wh/analytics/stock_daily", + "to_column": "sku", + "kind": "copy", + }]), + "and the other way round — reached from the child end, which is upstream" + ); +} + +/// A deployed dbt project in `folder`, declaring `parent`'s relation as a source +/// and deriving `child` from it. `orders → mart → secret_out` is three projects +/// chained through two shared relations. +async fn seed_neighbour_project( + db: &Pool, + folder: &str, + hash: i64, + parent: (&str, &str), + child: (&str, &str), +) { + let path = format!("f/{folder}/proj"); + sqlx::query!( + "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms) + VALUES ($1, $2, $2, '{}', '{}')", + WS, + folder + ) + .execute(db) + .await + .unwrap(); + sqlx::query!( + "INSERT INTO script (workspace_id, hash, path, summary, description, content, + created_by, language, lock) + VALUES ($1, $2, $3, '', '', 'profile: {}', 'test-user', 'dbt', '')", + WS, + hash, + path, + ) + .execute(db) + .await + .unwrap(); + sqlx::query!( + "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id, + resource_type, name, asset_path, tags) + VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000', 'source.q.' || $4, + 'source', $4, $5, '{}'), + ($1, $2, $3, '00000000-0000-0000-0000-000000000000', 'model.q.' || $6, + 'model', $6, $7, '{}')", + WS, + path, + hash, + parent.0, + parent.1, + child.0, + child.1, + ) + .execute(db) + .await + .unwrap(); + sqlx::query!( + "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id, + parent_unique_id, parent_column, child_unique_id, + child_column, lineage_kind) + VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000', + 'source.q.' || $4, 'k', 'model.q.' || $5, 'k', 'copy')", + WS, + path, + hash, + parent.0, + child.0, + ) + .execute(db) + .await + .unwrap(); +} + +/// A trace crosses out of the project it started in, and the gate crosses with +/// it. +/// +/// A relation one project produces is another's source, so the component reaches +/// edges the first project's owner set never named — that is what resolving +/// owners to a fixpoint is for. The other half is that the caller's access has +/// to be re-decided for each project discovered on the way: reaching a relation +/// says nothing about who may read the project on the far side of it. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn column_lineage_crosses_projects_only_where_the_caller_may_read_them(db: Pool) { + seed(&db, uuid::Uuid::from_u128(7)).await; + seed_deployed_orders(&db).await; + seed_neighbour_project( + &db, + "mid", + 43, + ("orders", "u/a/wh/analytics/orders"), + ("mart", "u/a/wh/analytics/mart"), + ) + .await; + seed_neighbour_project( + &db, + "secret", + 44, + ("mart", "u/a/wh/analytics/mart"), + ("secret_out", "u/a/wh/analytics/secret_out"), + ) + .await; + + let admin = ApiAuthed { is_admin: true, ..outsider() }; + let reached = |body: &serde_json::Value| { + body["edges"] + .as_array() + .unwrap() + .iter() + .map(|e| e["to_asset_path"].as_str().unwrap().to_string()) + .collect::>() + }; + + let all = column_lineage(&db, &admin, &["u/a/wh/analytics/orders"], None).await; + assert_eq!( + reached(&all), + [ + "u/a/wh/analytics/mart", + "u/a/wh/analytics/orders", + "u/a/wh/analytics/secret_out" + ] + .map(String::from) + .into(), + "two projects out from the one asked about, not one: {all}" + ); + + // Granted the first two folders and not the third. The edges of the project + // they may read are theirs; the one beyond it is not, even though the + // relation joining them is in the answer. + let partial = ApiAuthed { + folders: vec![ + ("private".to_string(), false, false), + ("mid".to_string(), false, false), + ], + ..outsider() + }; + let some = column_lineage(&db, &partial, &["u/a/wh/analytics/orders"], None).await; + assert_eq!( + reached(&some), + ["u/a/wh/analytics/mart", "u/a/wh/analytics/orders"] + .map(String::from) + .into(), + "the trace stops where the caller's access does: {some}" + ); + + // The other half of the same gate, and the half that is hand-written SQL + // rather than RLS: a token scoped to one folder reaches the projects in it + // and no others, whatever its grants say. + let scoped = ApiAuthed { + is_admin: true, + scopes: Some(vec!["scripts:read:f/private/*".to_string()]), + ..outsider() + }; + let scoped = column_lineage(&db, &scoped, &["u/a/wh/analytics/orders"], None).await; + assert_eq!( + reached(&scoped), + ["u/a/wh/analytics/orders"].map(String::from).into(), + "and where its scope does: {scoped}" + ); + + // A version pin answers for that version's project alone — the dbt editor, + // which draws one project as of one deploy. Crossing into `mid` here would + // annotate that canvas with relations it does not draw. + let pinned_version = column_lineage_q( + &db, + &admin, + vec![ + ( + "asset_path".to_string(), + "u/a/wh/analytics/orders".to_string(), + ), + ("dbt_script_hash".to_string(), format!("{:016x}", HASH)), + ], + None, + ) + .await; + assert_eq!( + reached(&pinned_version), + ["u/a/wh/analytics/orders"].map(String::from).into(), + "a version pin does not cross into the project beside it: {pinned_version}" + ); +} + +/// The bound on the answer, and that hitting it is said rather than silently +/// cut: a trace that stops reads exactly like one that ends. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn a_component_past_the_bound_is_cut_and_says_so(db: Pool) { + seed(&db, uuid::Uuid::from_u128(7)).await; + seed_deployed_orders(&db).await; + // One direct edge per column pair, more of them than a trace may carry. + sqlx::query!( + "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id, + parent_unique_id, parent_column, child_unique_id, + child_column, lineage_kind) + SELECT $1, $2, $3, '00000000-0000-0000-0000-000000000000', + 'model.p.raw_orders', 'c' || i, 'model.p.orders', 'c' || i, 'copy' + FROM generate_series(1, 6000) i", + WS, + PATH, + HASH, + ) + .execute(&db) + .await + .unwrap(); + + let admin = ApiAuthed { is_admin: true, ..outsider() }; + let body = column_lineage(&db, &admin, &["u/a/wh/analytics/orders"], None).await; + assert_eq!(body["edges"].as_array().unwrap().len(), 5000); + assert_eq!(body["truncated"], serde_json::json!(true)); +} diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index f6960126b1..e2bb8c6a1a 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -24445,6 +24445,68 @@ paths: schema: $ref: "#/components/schemas/AssetGraph" + /w/{workspace}/assets/column_lineage: + get: + summary: Column-level lineage of a set of dbt relations + description: > + The direct (`copy` / `mod`) column-to-column lineage the given relations' + columns sit in — the connected component around them, from the engine's + static analysis. Not their own edges, which would stop one hop out since + a column trace walks transitively, and not a whole project's, which + carries model families the selection cannot reach. + + Several relations, answered as one union, because one selection reaches + several: a script's output column can derive from columns of several dbt + models. Unpinned, the component crosses projects — a relation one project + produces is another's source — and the caller's access is decided again + for every project it reaches, so a trace ends where their grants do. A + pinned answer, by version here or by job on the run route, is one + project's. + + Its own endpoint rather than a field on the asset graph: the graph is + folder-wide and polled by a run page, while this is rendered for one + selection at a time. Empty for projects that did not opt into the + analysis pass (`column_lineage: true`), which is the ordinary case. The + indirect `scan` kind is stored but never served: it reaches every output + column of its model. + operationId: getDbtColumnLineage + tags: + - asset + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: asset_path + in: query + required: true + description: > + The `dbt://` relations whose lineage to return. Repeated, once per + relation, and answered as one union. + schema: + type: array + items: + type: string + - name: dbt_script_hash + in: query + description: > + The deployed version a view is drawing, when it is drawing one — the + dbt editor, which shows a single project as of a single deploy. A + version-pinned answer is that version's project alone, the same as a + job-pinned one, and only the unpinned answer crosses projects: a pin + says which stored graph is on screen, and another project's live + graph is not part of it. + + A run's or an editor buffer's own graph is not reachable here: that + pins to a job, and costs the job-read gate — see + `jobs/dbt_column_lineage/{id}`. + schema: + type: string + responses: + "200": + description: the relations' column-level lineage + content: + application/json: + schema: + $ref: "#/components/schemas/DbtColumnLineage" + /w/{workspace}/assets/macros: get: summary: List every workspace DuckDB macro (deployed `// macros` libraries) @@ -24636,6 +24698,49 @@ paths: schema: $ref: "#/components/schemas/AssetGraph" + /w/{workspace}/jobs/dbt_column_lineage/{id}: + get: + summary: Get relations' project column lineage as one run saw it + description: > + The same answer as `assets/column_lineage`, for the project version a + single job ran — including the dbt editor's parse of its own buffer, + whose graph belongs to that job and is reachable no other way. One + project answers here, the one the run is of, since the graph this + annotates is that project's too. Authorized through the job, the same + gate as `dbt_graph`. Reaching the run is not on its own enough to read + the project: a caller with no access to the script gets its relations and + `ref()` edges from `dbt_graph` and an empty answer here, exactly as that + endpoint redacts the model's SQL. + operationId: getDbtRunColumnLineage + tags: + - job + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: id + in: path + required: true + description: The job whose graph the lineage is read from + schema: + type: string + format: uuid + - name: asset_path + in: query + required: true + description: > + The `dbt://` relations whose lineage to return. Repeated, once per + relation, and answered as one union. + schema: + type: array + items: + type: string + responses: + "200": + description: the relations' column-level lineage + content: + application/json: + schema: + $ref: "#/components/schemas/DbtColumnLineage" + /w/{workspace}/jobs/run_progress/{id}: get: summary: List the per-relation progress one job has recorded so far @@ -26229,6 +26334,40 @@ components: drawn identically and the ambiguity would otherwise just move into the editor. Omitted for the unpinned workspace graph, which spans every project and so has no one time. + DbtColumnLineage: + type: object + description: >- + The direct column-to-column lineage the asked-for relations' columns sit + in — the connected component around them — in the terms the canvas draws: + relations and columns, never dbt's node ids. + required: [edges, truncated] + properties: + edges: + type: array + items: + type: object + required: [from_asset_path, from_column, to_asset_path, to_column, kind] + properties: + from_asset_path: + type: string + from_column: + type: string + to_asset_path: + type: string + to_column: + type: string + kind: + type: string + description: >- + dbt's own word for how the value travelled — `copy` + (passthrough) or `mod` (transformed). Not an enum: the engine + treats the set as open. + truncated: + type: boolean + description: >- + The component reaches further than `edges`, which holds the part + nearest the asked-for relations. A trace that stops short is + otherwise indistinguishable from one that ends. DbtAssetProvenance: type: object description: >- diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index dbf33cdddc..83a92bcd3c 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -139,6 +139,7 @@ pub fn workspaced_service() -> Router { .route("/run_progress/{id}", get(get_run_progress)) .route("/run_assets/{id}", get(list_run_assets)) .route("/dbt_graph/{id}", get(get_dbt_run_graph)) + .route("/dbt_column_lineage/{id}", get(get_dbt_run_column_lineage)) .route("/dbt_resumable/{id}", get(get_dbt_resumable)) .route( "/dbt_resumable_script/p/{*script_path}", @@ -891,21 +892,27 @@ struct AssetProgress { error: Option, } -/// The asset graph as one run saw it. Pinning to a job needs the full job-read -/// contract, so it lives on `require_job_read_access` here rather than as a -/// parameter on `/assets/graph`. See docs/dbt-runtime.md. -async fn get_dbt_run_graph( - authed: ApiAuthed, - OptViewToken(view_token): OptViewToken, - Extension(db): Extension, - Extension(user_db): Extension, - Path((w_id, job_id)): Path<(String, Uuid)>, - Query(q): Query, -) -> error::JsonResult { +/// Which project version a dbt view pins to for this job, once the caller has +/// been shown to be entitled to it. +/// +/// `Ok(None)` is "answer unpinned", not a refusal: a job that stored no graph of +/// its own — and one that has aged out of retention — is served the deployed +/// version rather than an error, so a run page keeps drawing after the run is +/// gone. Pinning needs the full job-read contract, which is why it lives on +/// `require_job_read_access` here rather than as a parameter on `/assets/*`. +/// See docs/dbt-runtime.md. +async fn dbt_pinned_run( + authed: &ApiAuthed, + db: &DB, + user_db: &UserDB, + w_id: &str, + job_id: Uuid, + view_token: Option<&str>, +) -> error::Result> { // The scope domain comes from the URL segment, so `/jobs` asks a scoped token // for `jobs:read` alone while the body returned is asset data. Both are // required: the job gate below reaches this run, this reaches assets at all. - check_scopes(&authed, || "assets:read".to_string())?; + check_scopes(authed, || "assets:read".to_string())?; let job = sqlx::query!( r#"SELECT created_by, runnable_path, CASE WHEN kind = 'script' THEN runnable_id END AS script_hash, @@ -918,42 +925,70 @@ async fn get_dbt_run_graph( AND g.script_hash IS NULL) AS "editor_graph!" FROM v2_job WHERE id = $1 AND workspace_id = $2"#, job_id, - &w_id + w_id ) - .fetch_optional(&db) + .fetch_optional(db) .await?; - // No such job: answer the unpinned graph rather than 404, so a run page whose - // job has aged out of retention still draws the deployed version instead of - // an error. Reachable only with `assets:read`, which is exactly what - // `/assets/graph` would have cost for the same answer. + // Unpinned rather than 404 for a job that is gone. Reachable only with + // `assets:read`, which is exactly what the unpinned route would have cost + // for the same answer. let Some(job) = job else { - return windmill_api_assets::asset_graph_for(&authed, &w_id, user_db, db, q, None).await; + return Ok(None); }; require_job_read_access( - &db, - &user_db, - &authed, - &w_id, + db, + user_db, + authed, + w_id, &job_id, &job.created_by, - view_token.as_deref(), + view_token, ) .await?; // A preview or flow job names no deployed version, so there is usually no // graph to pin to and the workspace one answers. The exception is a job that // parsed one itself, which is what the dbt editor's refresh is: its graph // belongs to that job alone and nothing else can reach it. - let pinned = job + Ok(job .runnable_path .filter(|_| job.script_hash.is_some() || job.editor_graph) .map(|path| windmill_api_assets::PinnedRun { job_id, script_path: path, script_hash: job.script_hash, - }); + })) +} + +/// The asset graph as one run saw it. +async fn get_dbt_run_graph( + authed: ApiAuthed, + OptViewToken(view_token): OptViewToken, + Extension(db): Extension, + Extension(user_db): Extension, + Path((w_id, job_id)): Path<(String, Uuid)>, + Query(q): Query, +) -> error::JsonResult { + let pinned = + dbt_pinned_run(&authed, &db, &user_db, &w_id, job_id, view_token.as_deref()).await?; windmill_api_assets::asset_graph_for(&authed, &w_id, user_db, db, q, pinned).await } +/// The column lineage a set of relations sits in as one run saw it — the same +/// pin as `get_dbt_run_graph`, for the trace drawn beside a node of that graph. +async fn get_dbt_run_column_lineage( + authed: ApiAuthed, + OptViewToken(view_token): OptViewToken, + Extension(db): Extension, + Extension(user_db): Extension, + Path((w_id, job_id)): Path<(String, Uuid)>, + Query(pairs): Query>, +) -> error::JsonResult { + let q = windmill_api_assets::ColumnLineageQuery::from_query_pairs(pairs)?; + let pinned = + dbt_pinned_run(&authed, &db, &user_db, &w_id, job_id, view_token.as_deref()).await?; + windmill_api_assets::dbt_column_lineage_for(&authed, &w_id, user_db, q, pinned).await +} + /// Whether a `dbt retry` submitted by this caller would resume THIS run. /// /// One failure is saved per script per execution principal, so a page showing an diff --git a/docs/dbt-runtime.md b/docs/dbt-runtime.md index 2414c49cc4..926948a676 100644 --- a/docs/dbt-runtime.md +++ b/docs/dbt-runtime.md @@ -435,7 +435,9 @@ Two things make that safe rather than a widening: model set and its relations are already visible to them. - **`raw_code` is gated separately**, on an `EXISTS` against `script` in the authed transaction. The body of a model is the project's source code and stays - behind access to the project, whatever the shape query resolved. + behind access to the project, whatever the shape query resolved. `column_schema` + and the column trace behind `/jobs/dbt_column_lineage/{id}` take the same gate, + for the same reason: both are the shape of what the author wrote. The path and hash coming from the job row rather than the query also means a caller cannot pin one project's version while naming another's run. @@ -1647,19 +1649,66 @@ table is `ref()` lineage. The typed column list lands in `dbt_node.column_schema`, beside `columns` rather than merged into it — `columns` stays what the author *declared*. -**Stored now, served later.** This change lands the ingest and the storage; the -endpoint that draws a column trace is a follow-up. What is user-visible today is -`column_schema` — every column of a relation, typed and in the order the model -emits them — which rides the asset graph the details pane already fetches, and -replaces a panel that could only list the columns an author happened to document. -The edges sit in `dbt_column_edge` waiting for their surface. +Two things are user-visible. `column_schema` — every column of a relation, typed +and in the order the model emits them — rides the asset graph the details pane +already fetches, and replaces a panel that could only list the columns an author +happened to document. The edges are served by an endpoint of their own, +`assets/column_lineage`, which the pane asks for the selection it is drawing. -`column_schema` is gated on being able to read the producing project, like the -model's SQL: a column-level view is the shape of what the author wrote, one level -finer than the `ref()` graph, which is ungated only because it draws relations the -caller already sees. A share-link viewer entitled to a dbt run therefore gets its +Both are gated on being able to read the producing project, like the model's SQL: +a column-level view is the shape of what the author wrote, one level finer than +the `ref()` graph, which is ungated only because it draws relations the caller +already sees. A share-link viewer entitled to a dbt run therefore gets its relations and `ref()` edges, and neither the SQL nor the columns. +**One request per selection, and the gate re-decided per project.** The endpoint +takes every relation the view has reached and answers their union, because one +selection reaches several — a script's output column can derive from columns of +several models. Holding partial answers between selections instead was tried and +is what a client cache is: it produced a wrong premise for a relation two projects +describe, then staleness on redeploy, then a lost retry. + +The answer is the connected component around those relations, and that component +does not stop at the project that owns them: a relation one project produces is +another's source, so the walk resolves owners, reads their edges, walks, and +repeats for the relations that walk newly reached. Resolving once — for the +relations asked about — stops the trace at the first project boundary. The +security half is that a project reached this way is a project the caller may not +be entitled to, so the scope filter and the project's visibility are re-applied to +every project the expansion discovers, not decided once for the first owner set. + +A PINNED answer is the exception and needs none of it, whether it names a job or +a deployed version: the pin says which stored graph is on screen, and another +project's live graph is not part of it, so it answers for that one project. The +dbt editor pins by version on every selection and the run page pins by job; the +pipeline page pins nothing, and is where the walk crosses projects. + +**The component is bounded, and says when it was cut.** The renderer draws a box +per column, so a component past a few thousand edges is unreadable however it is +served — a synthetic 3000-model project whose models share a column returns 58k +direct edges and 7.3MB. The walk is breadth-first from the asked-for relations and +stops at 5000 edges, so what survives is the part nearest the selection rather +than an arbitrary slice, and `truncated` says so: a trace that stops short is +otherwise indistinguishable from one that ends. The FETCH is not bounded the same +way — a project's edges arrive whole, because the walk is what decides which of +them are in the component, and a `LIMIT` would cut a set that need not contain +the asked-for relation at all. What bounds it is the ingest's own cap per version +plus a stop on discovering further projects once the held set is outsized. + +The walk is in Rust rather than a recursive CTE. `EXPLAIN ANALYZE` on that same +project measured 1243ms against 59ms for the query alone: a CTE has no index to +walk, so the recursive term rescans the doubled edge set once per level (11.7M +rows), while the same walk over a map is microseconds. + +The two halves of a trace — dbt's and the pipeline's — meet at shared node ids. A +DuckDB script's `// column x <- dbt://wh/s/model.col` mints the same +`(dbt, path, column)` node dbt's own lineage does, so the producer graph the asset +graph already carries and the dbt graph are MERGED rather than chosen between, and +a trace crosses that boundary in either direction. What the browser cannot close +in one request is a relation the server discovers whose columns are consumed by a +script that writes into a third project: the producer half of that hop is the +canvas's, not the server's, and the seeds were computed before the answer arrived. + **The analysis pass takes the build's own `--full-refresh`.** `is_incremental()` branches on it, so an incremental model reading `{{ this }}` compiles its self-join — and any `ref()` inside that branch — only when the flag is absent. A @@ -1691,7 +1740,7 @@ the compile that produced it, and a run that re-ingests describes its own run. | `unique`/`not_null`/`accepted_values`/`relationships` | `data_tests` | exact 1:1 with the four `// data_test` kinds | | declared column metadata | `columns` on the asset node | descriptions only, from the manifest | | analyzed column schema | `column_schema` on the asset node | `dbt.node_columns.parquet`, opt-in | -| column-to-column lineage | `dbt_column_edge` rows (no view yet) | `dbt.column_lineage.parquet`, opt-in | +| column-to-column lineage | `dbt_column_edge` rows, drawn as a column trace | `dbt.column_lineage.parquet`, opt-in | | model `tags` | node badge | `tag` | | source freshness | `freshness` | `last_success_at` chip | | `run_results.json` | materialization records | `record_materialization` | diff --git a/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte b/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte index 2c5e5fd97e..005e3a3ffb 100644 --- a/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte @@ -36,7 +36,8 @@ type ColumnLineage, type PipelineAnnotations } from './parsePipelineAnnotations' - import ColumnLineageTrace from './ColumnLineageTrace.svelte' + import ColumnTraceSection from './ColumnTraceSection.svelte' + import DbtColumnList from './DbtColumnList.svelte' import { extractDraftMacros } from './resolveGraph' import { assetColumnNodes, type ColumnLineageGraph } from './columnLineageGraph' import SummaryPathDisplay from '$lib/components/SummaryPathDisplay.svelte' @@ -156,6 +157,12 @@ // resolved graph). Drives the transitive column-lineage trace shown for a // selected materialized asset. selectionColumnGraph?: ColumnLineageGraph + /** That graph still being fetched — a dbt relation's lineage is a request + * of its own, so it arrives after the selection does. */ + selectionColumnLoading?: boolean + /** The lineage reaches past what the graph holds: the API cut it at the + * part nearest the selection. */ + selectionColumnTruncated?: boolean /** dbt provenance of the selected relation, when a dbt project * materializes it — carries the model's own SQL. */ selectionDbt?: DbtAssetProvenance @@ -289,6 +296,8 @@ onScriptRemoved, selectionProducers = [], selectionColumnGraph, + selectionColumnLoading = false, + selectionColumnTruncated = false, selectionDbt, schemaCanEvolve = true, selectionForkMaterialization = undefined, @@ -446,6 +455,19 @@ return scripts.length === 1 ? `${scripts[0].path}__dbt/${file}` : file }) + // The two things a dbt relation can show besides its SQL, and what decides + // whether the panel opens at all for one that has none: the columns the model + // produces, and the trace they sit in. A share-link viewer gets neither — + // both are gated on reading the project, like the SQL. + let selectionDbtHasColumns = $derived( + !!selectionDbt?.column_schema?.length || Object.keys(selectionDbt?.columns ?? {}).length > 0 + ) + let selectionColumnNodes = $derived( + selection?.kind === 'asset' && selectionColumnGraph + ? assetColumnNodes(selectionColumnGraph, selection.asset_kind, selection.path) + : [] + ) + // Bound from ScriptEditor — populated by inferAssets on every code // change. Forwarded to the page so the canvas can re-derive write // edges as the user edits the body (e.g. renaming a CREATE TABLE @@ -1221,22 +1243,20 @@
{/if} - {#if selectionColumnGraph && assetColumnNodes(selectionColumnGraph, selection.asset_kind, selection.path).length > 0} -
- -
- {/if} +
{/key} - {:else if selectionDbt?.raw_code} + {:else if selectionDbt && (selectionDbt.raw_code || selectionDbtHasColumns || selectionColumnNodes.length > 0 || selectionColumnLoading)} + {#if selectionDbtHasColumns} +
+ +
+ {/if} + + {#if selectionDbt.raw_code} +
+ +
+ {/if} {:else}
diff --git a/frontend/src/lib/components/assets/AssetGraph/ColumnTraceSection.svelte b/frontend/src/lib/components/assets/AssetGraph/ColumnTraceSection.svelte new file mode 100644 index 0000000000..0422b88cc8 --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/ColumnTraceSection.svelte @@ -0,0 +1,50 @@ + + +{#if loading && nodes.length === 0} +
+ + Loading column lineage +
+{:else if graph && nodes.length > 0} +
+ + {#if truncated} +
+ Showing the part of the trace nearest this relation. The lineage reaches further than one + view can draw. +
+ {/if} +
+{/if} diff --git a/frontend/src/lib/components/assets/AssetGraph/DbtColumnList.svelte b/frontend/src/lib/components/assets/AssetGraph/DbtColumnList.svelte new file mode 100644 index 0000000000..a1c769d9af --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/DbtColumnList.svelte @@ -0,0 +1,50 @@ + + +{#if columns.length > 0} +
+
{analyzed ? 'columns' : 'columns declared'}
+
+ {#each columns as col (col.name)} +
+ {col.name} + {#if col.type} + {col.type} + {/if} + {col.description} +
+ {/each} +
+ + {#if !analyzed} +
+ Declared metadata. Set `column_lineage: true` in the descriptor for the real column schema, + typed and in the order the model produces it. +
+ {/if} +
+{/if} diff --git a/frontend/src/lib/components/assets/AssetGraph/PipelineGraphEditor.svelte b/frontend/src/lib/components/assets/AssetGraph/PipelineGraphEditor.svelte index 3439fa5ad5..721fafb6be 100644 --- a/frontend/src/lib/components/assets/AssetGraph/PipelineGraphEditor.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/PipelineGraphEditor.svelte @@ -17,7 +17,9 @@ AssetGraphResponse, AssetGraphSelection, NativeTriggerKind, - PipelineMode, DbtAssetProvenance } from './types' + PipelineMode, + DbtAssetProvenance + } from './types' import type { AssetKind, Script, ScriptLang } from '$lib/gen' import type { RunnableRunState, PipelineEvent } from './activeRunnables.svelte' import type { PipelineOutputKind } from './pipelineTemplates' @@ -76,6 +78,8 @@ localScriptsVersion, selectionProducers = [], selectionColumnGraph, + selectionColumnLoading = false, + selectionColumnTruncated = false, selectionDbt, schemaCanEvolve = true, selectionForkMaterialization = undefined, @@ -180,8 +184,13 @@ * the selected node's source on live-reload. */ localScriptsVersion?: unknown selectionProducers?: Array<{ kind: 'script' | 'flow'; path: string; unsaved?: boolean }> - /** Transitive column-lineage trace for a selected ducklake asset (route page). */ + /** Transitive column-lineage trace for the selected asset (route page). */ selectionColumnGraph?: ColumnLineageGraph + /** That trace still being fetched — a dbt relation's is a request of its + * own, so it arrives after the selection does. */ + selectionColumnLoading?: boolean + /** That trace cut at the part nearest the selection. */ + selectionColumnTruncated?: boolean /** dbt provenance of the selected relation — carries its SQL. */ selectionDbt?: DbtAssetProvenance schemaCanEvolve?: boolean @@ -514,6 +523,8 @@ selection={activeDraft ? undefined : editor.selection} selectionProducers={activeDraft ? [] : selectionProducers} {selectionColumnGraph} + {selectionColumnLoading} + {selectionColumnTruncated} {selectionDbt} {schemaCanEvolve} {selectionForkMaterialization} diff --git a/frontend/src/lib/components/assets/AssetGraph/columnLineageGraph.test.ts b/frontend/src/lib/components/assets/AssetGraph/columnLineageGraph.test.ts index 6842dcfd86..16c8713c5c 100644 --- a/frontend/src/lib/components/assets/AssetGraph/columnLineageGraph.test.ts +++ b/frontend/src/lib/components/assets/AssetGraph/columnLineageGraph.test.ts @@ -2,7 +2,10 @@ import { describe, expect, it } from 'vitest' import type { AssetGraphResponse } from './types' import { buildColumnGraph, + buildDbtColumnGraph, colNodeId, + mergeColumnGraphs, + type ColumnLineageGraph, traceColumn, connectedComponent, assetColumnNodes, @@ -120,6 +123,79 @@ describe('buildColumnGraph', () => { }) }) +describe('buildDbtColumnGraph', () => { + it('takes the direct kinds and drops any other', () => { + // `scan` means the column was read to produce the ROW — a join key, a + // predicate, a `group by` — so it reaches every output column of its model + // and is not what a column trace means. The server filters it out; this + // filters again, because the kind set is the engine's and an unknown one + // must not become an edge the trace calls data flow. + const g = buildDbtColumnGraph([ + { + from_asset_path: 'main/s/stg', + from_column: 'raw_name', + to_asset_path: 'main/s/mart', + to_column: 'clean_name', + kind: 'mod' + }, + { + from_asset_path: 'main/s/stg', + from_column: 'id', + to_asset_path: 'main/s/mart', + to_column: 'id', + kind: 'copy' + }, + { + from_asset_path: 'main/s/stg', + from_column: 'id', + to_asset_path: 'main/s/mart', + to_column: 'clean_name', + kind: 'scan' + } + ]) + expect(g.up.get(colNodeId('dbt', 'main/s/mart', 'clean_name'))).toEqual( + new Set([colNodeId('dbt', 'main/s/stg', 'raw_name')]) + ) + expect(g.up.get(colNodeId('dbt', 'main/s/mart', 'id'))).toEqual( + new Set([colNodeId('dbt', 'main/s/stg', 'id')]) + ) + }) +}) + +describe('mergeColumnGraphs', () => { + it('chains a dbt column into what a producer derives from it', () => { + // The two halves arrive separately — the producer's from the asset graph, + // dbt's from its own request — and meet at the dbt node a `// column` + // annotation names. A trace has to cross that, or a dbt selection stops + // before the script consuming it. + const dbt = buildDbtColumnGraph([ + { + from_asset_path: 'main/s/stg', + from_column: 'raw', + to_asset_path: 'main/s/mart', + to_column: 'clean', + kind: 'copy' + } + ]) + const producer: ColumnLineageGraph = { + nodes: new Map(), + up: new Map(), + down: new Map() + } + const src = colNodeId('dbt', 'main/s/mart', 'clean') + const out = colNodeId('ducklake', 'wh/report', 'total') + producer.nodes.set(src, { kind: 'dbt', path: 'main/s/mart', column: 'clean' }) + producer.nodes.set(out, { kind: 'ducklake', path: 'wh/report', column: 'total' }) + producer.up.set(out, new Set([src])) + producer.down.set(src, new Set([out])) + + const merged = mergeColumnGraphs(dbt, producer) + expect(traceColumn(colNodeId('dbt', 'main/s/stg', 'raw'), merged)).toEqual( + new Set([colNodeId('dbt', 'main/s/stg', 'raw'), src, out]) + ) + }) +}) + describe('traceColumn', () => { it('returns the full upstream + downstream impact set of a source column', () => { const g = buildColumnGraph(chainGraph()) diff --git a/frontend/src/lib/components/assets/AssetGraph/columnLineageGraph.ts b/frontend/src/lib/components/assets/AssetGraph/columnLineageGraph.ts index 50fa1c4e7b..54837753e1 100644 --- a/frontend/src/lib/components/assets/AssetGraph/columnLineageGraph.ts +++ b/frontend/src/lib/components/assets/AssetGraph/columnLineageGraph.ts @@ -1,6 +1,11 @@ -import type { AssetKind } from '$lib/gen' +import type { AssetKind, DbtColumnLineage } from '$lib/gen' import type { AssetGraphResponse } from './types' +// One column-to-column edge of a dbt project's static analysis, as the API +// serves it. Taken from the generated client rather than restated: unlike the +// asset graph, this response is fetched through it. +export type DbtColumnEdge = DbtColumnLineage['edges'][number] + // A node in the column-level lineage graph: one column of one asset. export type ColumnNode = { kind: AssetKind; path: string; column: string } export type ColumnNodeId = string @@ -23,6 +28,21 @@ export type ColumnLineageGraph = { down: Map> } +export const EMPTY_COLUMN_GRAPH: ColumnLineageGraph = { + nodes: new Map(), + up: new Map(), + down: new Map() +} + +// Direct value flow, as dbt's static analysis labels it: `copy` passes a column +// through, `mod` transforms it. The API serves only those two — the third kind, +// `scan`, means the column was read to produce the ROW rather than the value (a +// join key, a `where` predicate, a `group by`), so it reaches EVERY output +// column of the model and would draw the diagram as a complete bipartite graph. +// Filtered here as well so a kind the engine invents cannot silently become an +// edge the trace claims is data flow. +const DIRECT_DBT_LINEAGE = new Set(['copy', 'mod']) + // Build the column graph from a resolved asset graph. A producer's // `column_lineage` describes the columns of the asset it materializes; that // output asset is the ducklake target it writes (v1 materialize target), found @@ -89,6 +109,59 @@ export function buildColumnGraph(graph: AssetGraphResponse): ColumnLineageGraph return { nodes, up, down } } +// The same graph, from dbt's own column lineage. dbt arrives already resolved to +// two relations rather than anchored to a producer, and the API serves only the +// direct kinds, so this is a straight edge list. +export function buildDbtColumnGraph(edges: DbtColumnEdge[]): ColumnLineageGraph { + const nodes = new Map() + const up = new Map>() + const down = new Map>() + const addNode = (n: ColumnNode): ColumnNodeId => { + const id = colNodeId(n.kind, n.path, n.column) + if (!nodes.has(id)) nodes.set(id, n) + return id + } + for (const e of edges) { + // Belt and braces: the API filters to `copy`/`mod`, and a kind an engine + // invents must not silently become an edge the trace calls data flow. + if (!DIRECT_DBT_LINEAGE.has(e.kind)) continue + const src = addNode({ kind: 'dbt', path: e.from_asset_path, column: e.from_column }) + const out = addNode({ kind: 'dbt', path: e.to_asset_path, column: e.to_column }) + if (src === out) continue + ;(up.get(out) ?? up.set(out, new Set()).get(out)!).add(src) + ;(down.get(src) ?? down.set(src, new Set()).get(src)!).add(out) + } + return { nodes, up, down } +} + +// One graph out of several, so a trace crosses the boundary between them. +// +// The two halves reach each other through shared node ids: a producer's +// `// column out <- dbt://wh/schema/model.col` puts a `('dbt', path, column)` +// node in the producer graph under the same `colNodeId` the dbt lineage mints +// for it, so the union chains a dbt model's columns into the script that +// consumes them and on into what that script writes. Kept separate up to here +// because they are fetched separately — the producer half rides on the asset +// graph, the dbt half is asked for per selection. +export function mergeColumnGraphs(...graphs: ColumnLineageGraph[]): ColumnLineageGraph { + const nodes = new Map() + const up = new Map>() + const down = new Map>() + for (const g of graphs) { + for (const [id, n] of g.nodes) if (!nodes.has(id)) nodes.set(id, n) + for (const [dir, into] of [ + [g.up, up], + [g.down, down] + ] as const) { + for (const [id, adj] of dir) { + const target = into.get(id) ?? into.set(id, new Set()).get(id)! + for (const m of adj) target.add(m) + } + } + } + return { nodes, up, down } +} + // Every node reachable from `start` by following `adj` (transitive closure, // excluding `start` itself). Iterative to avoid deep-recursion limits. function reach(start: ColumnNodeId, adj: Map>): Set { diff --git a/frontend/src/lib/components/assets/AssetGraph/dbtColumnLineage.svelte.ts b/frontend/src/lib/components/assets/AssetGraph/dbtColumnLineage.svelte.ts new file mode 100644 index 0000000000..189ba05b5d --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/dbtColumnLineage.svelte.ts @@ -0,0 +1,120 @@ +import { AssetService, JobService, type DbtColumnLineage } from '$lib/gen' +import { + buildDbtColumnGraph, + EMPTY_COLUMN_GRAPH, + type ColumnLineageGraph +} from './columnLineageGraph' + +/** Which stored dbt graph a view is drawing. A job — a run's snapshot, or the + * editor's parse of its own buffer — is asked through the job route, the only + * way to reach a graph that names no deployed version; otherwise the deployed + * version by hash, or the current one when there is no hash. */ +export type DbtGraphPin = { jobId?: string; scriptHash?: string | number } + +/** What a selection's dbt column lineage is doing right now. `loading` is + * separate because a project still being fetched and one that never asked for + * the analysis pass are the same empty graph otherwise. */ +export type DbtColumnLineageState = { + readonly graph: ColumnLineageGraph + readonly loading: boolean + /** The component reaches past what `graph` holds — the API cut it at the + * part nearest the selection. */ + readonly truncated: boolean +} + +function fetchLineage( + workspace: string, + assetPaths: string[], + pin: DbtGraphPin | undefined +): Promise { + return pin?.jobId + ? JobService.getDbtRunColumnLineage({ workspace, id: pin.jobId, assetPath: assetPaths }) + : AssetService.getDbtColumnLineage({ + workspace, + assetPath: assetPaths, + dbtScriptHash: pin?.scriptHash != undefined ? String(pin.scriptHash) : undefined + }) +} + +/** Follow the selection, fetching the dbt column lineage it reaches. + * + * One request per selection, whatever it reaches: the API takes every relation + * at once and walks out from all of them, so there is no partial answer to hold + * on to between selections and nothing to go stale behind a redeploy. + * + * Per selection rather than off the graph response: the graph is folder-wide + * and a run page polls it, while this is drawn for one selection. It also means + * the request is never made for a project that did not opt into the analysis + * pass — the pane simply never shows the section. + */ +export function useDbtColumnLineage(args: { + workspace: () => string | undefined + /** The dbt relations to expand. The selection itself when it is one; for a + * selection of another kind, every dbt relation its own lineage reaches — + * a ducklake table can be derived from several, and expanding only the + * first would leave the rest as leaves. */ + assetPaths: () => string[] + /** The graph on screen, so the lineage describes the same project. */ + pin?: () => DbtGraphPin | undefined +}): DbtColumnLineageState { + let graph = $state(EMPTY_COLUMN_GRAPH) + let loading = $state(false) + let truncated = $state(false) + + // The question the state in hand answers, and a counter deciding which answer + // is still wanted. Neither is a cache of edges: the API returns a whole + // component, so an answer is either the current selection's or nothing. + let asked: string | undefined = undefined + let latest = 0 + + $effect(() => { + const workspace = args.workspace() + const paths = [...new Set(args.assetPaths())].sort() + const pin = args.pin?.() + const question = JSON.stringify([workspace, pin?.jobId, pin?.scriptHash, paths]) + // A selection re-derived from a graph that polled is the same question. Not + // asking it again is what keeps a run page from refetching a component's + // worth of edges every poll to redraw what is already on screen. + if (question === asked) return + asked = question + const id = ++latest + if (!workspace || paths.length === 0) { + graph = EMPTY_COLUMN_GRAPH + truncated = false + loading = false + return + } + loading = true + fetchLineage(workspace, paths, pin).then( + (r) => { + if (id !== latest) return + graph = buildDbtColumnGraph(r?.edges ?? []) + truncated = r?.truncated ?? false + loading = false + }, + // Lineage annotates a graph that renders without it, so a failed fetch + // leaves that branch unexpanded rather than putting an error over the + // model. Not retried on its own: the effect reruns whenever the canvas + // does, and a failing endpoint would then be asked once per redraw. + // Selecting another node and back asks again. + () => { + if (id !== latest) return + graph = EMPTY_COLUMN_GRAPH + truncated = false + loading = false + } + ) + }) + + return { + get graph() { + return graph + }, + get loading() { + return loading + }, + get truncated() { + return truncated + } + } +} diff --git a/frontend/src/lib/components/dbt/DbtEditor.svelte b/frontend/src/lib/components/dbt/DbtEditor.svelte index 1d3de73e58..469536f305 100644 --- a/frontend/src/lib/components/dbt/DbtEditor.svelte +++ b/frontend/src/lib/components/dbt/DbtEditor.svelte @@ -31,6 +31,15 @@ AssetGraphNodeData, DbtAssetProvenance } from '$lib/components/assets/AssetGraph/types' + import { + useDbtColumnLineage, + type DbtGraphPin + } from '$lib/components/assets/AssetGraph/dbtColumnLineage.svelte' + import { + EMPTY_COLUMN_GRAPH, + mergeColumnGraphs, + type ColumnLineageGraph + } from '$lib/components/assets/AssetGraph/columnLineageGraph' import { DBT_DESCRIPTOR, DBT_MODULE_EXTENSIONS, @@ -223,6 +232,28 @@ // the deployed graph, which previews by version instead. Either way the rows // come from the project whose SQL is displayed above them. let selectedBuffer = $state(undefined) + // Which graph the selection came from, so the lineage fetched below is the + // selected node's own project rather than whatever is deployed. + let selectionPin = $state(undefined) + // The selected model's column lineage, fetched on selection. Its own request + // rather than a field on the graph: only a project that opted into the + // analysis pass has any, and it is drawn for one model at a time. + const columnLineage = useDbtColumnLineage({ + workspace: () => opWs, + assetPaths: () => { + const path = selectedDbt ? selectedAsset?.path : undefined + return path ? [path] : [] + }, + pin: () => selectionPin + }) + // What the scripts around this project declare about its columns, off the + // same graph response the canvas drew. Merged rather than chosen between: a + // model's column and the ducklake column a script derives from it are one + // chain, and the trace has to cross that boundary. + let selectionProducerColumns = $state(EMPTY_COLUMN_GRAPH) + let selectionColumnGraph = $derived( + mergeColumnGraphs(columnLineage.graph, selectionProducerColumns) + ) let jobLoader: JobLoader | undefined = $state(undefined) let testJob: any = $state(undefined) @@ -525,10 +556,12 @@ testRunning={testIsLoading} testResult={testJob?.result} selection={graphSelection} - onSelect={(sel, dbt, buffer) => { + onSelect={(sel, dbt, buffer, pin, producerColumns) => { graphSelection = sel selectedDbt = dbt selectedBuffer = buffer + selectionPin = pin + selectionProducerColumns = producerColumns }} /> @@ -550,6 +583,9 @@ {args} fileInBundle={!!selectedDbt.original_file_path && !!modules?.[selectedDbt.original_file_path]} + columnGraph={selectionColumnGraph} + columnLoading={columnLineage.loading} + columnTruncated={columnLineage.truncated} onOpenFile={open} onClose={() => (graphSelection = undefined)} /> diff --git a/frontend/src/lib/components/dbt/DbtModelDetails.svelte b/frontend/src/lib/components/dbt/DbtModelDetails.svelte index 5067902c4c..d3dd25d04a 100644 --- a/frontend/src/lib/components/dbt/DbtModelDetails.svelte +++ b/frontend/src/lib/components/dbt/DbtModelDetails.svelte @@ -12,6 +12,9 @@ import { ClipboardCopy, Code2, FileCode2, Loader2, TableProperties, X } from 'lucide-svelte' import { copyToClipboard } from '$lib/utils' import type { DbtAssetProvenance } from '$lib/components/assets/AssetGraph/types' + import ColumnTraceSection from '$lib/components/assets/AssetGraph/ColumnTraceSection.svelte' + import DbtColumnList from '$lib/components/assets/AssetGraph/DbtColumnList.svelte' + import type { ColumnLineageGraph } from '$lib/components/assets/AssetGraph/columnLineageGraph' import { previewDbtRows, type DbtPreview, type DbtPreviewBuffer } from './previewRows' import { nodeSelector } from './parseDbtRun' @@ -34,6 +37,12 @@ args, /** Whether this model's file is in the bundle being edited. */ fileInBundle = false, + /** The project's column-level lineage, when the descriptor asked for it. + * Fetched for this relation against the same graph the canvas draws, so + * the trace and the nodes above it describe one parse. */ + columnGraph, + columnLoading = false, + columnTruncated = false, onOpenFile, onClose }: { @@ -45,6 +54,9 @@ buffer?: DbtPreviewBuffer args?: Record fileInBundle?: boolean + columnGraph?: ColumnLineageGraph + columnLoading?: boolean + columnTruncated?: boolean onOpenFile?: (path: string) => void onClose?: () => void } = $props() @@ -111,24 +123,9 @@ return typeof v === 'object' ? JSON.stringify(v) : String(v) } - // The real columns where the analysis pass produced them — typed and in the - // order the model emits them — and the declared ones otherwise. The - // description comes from `columns` either way: that is the only place an - // author's prose lives, and a project documents a handful of forty. - let columns = $derived( - dbt.column_schema?.length - ? dbt.column_schema.map((c) => ({ - name: c.name, - type: c.type, - description: dbt.columns?.[c.name] ?? '' - })) - : Object.entries(dbt.columns ?? {}).map(([name, description]) => ({ - name, - type: undefined, - description - })) + let hasColumns = $derived( + !!dbt.column_schema?.length || Object.keys(dbt.columns ?? {}).length > 0 ) - let columnsAreAnalyzed = $derived(!!dbt.column_schema?.length) // `dbt show` SELECTs from the node's own relation and the worker intersects // the selector with `resource_type:model`, so offering it on a seed, snapshot // or source only ever produces a failed job. @@ -251,35 +248,9 @@ {/if}
- {#if columns.length > 0 || (dbt.data_tests?.length ?? 0) > 0} + {#if hasColumns || (dbt.data_tests?.length ?? 0) > 0}
- {#if columns.length > 0} -
-
- {columnsAreAnalyzed ? 'columns' : 'columns declared'} -
-
- {#each columns as col (col.name)} -
- {col.name} - {#if col.type} - {col.type} - {/if} - {col.description} -
- {/each} -
- - {#if !columnsAreAnalyzed} -
- Declared metadata. Set `column_lineage: true` in the descriptor for the real - column schema, typed and in the order the model produces it. -
- {/if} -
- {/if} + {#if (dbt.data_tests?.length ?? 0) > 0}
tests
@@ -295,6 +266,15 @@
{/if} + + {#if showRows && preview} {#if 'error' in preview}
{preview.error}
diff --git a/frontend/src/lib/components/dbt/DbtModelGraph.svelte b/frontend/src/lib/components/dbt/DbtModelGraph.svelte index e6d948d926..bd7f42a9e7 100644 --- a/frontend/src/lib/components/dbt/DbtModelGraph.svelte +++ b/frontend/src/lib/components/dbt/DbtModelGraph.svelte @@ -25,6 +25,12 @@ DbtAssetProvenance } from '$lib/components/assets/AssetGraph/types' import { useDbtRunStatus } from './runStatus.svelte' + import type { DbtGraphPin } from '$lib/components/assets/AssetGraph/dbtColumnLineage.svelte' + import { + buildColumnGraph, + EMPTY_COLUMN_GRAPH, + type ColumnLineageGraph + } from '$lib/components/assets/AssetGraph/columnLineageGraph' let { workspace, @@ -80,7 +86,19 @@ * buffer rather than a deployed version — as submitted, not as the * editor holds it now. Sent with the selection rather than exposed on * its own so it can never disagree with the SQL the parent shows. */ - buffer: DbtPreviewBuffer | undefined + buffer: DbtPreviewBuffer | undefined, + /** Which graph this node was taken from, so anything else fetched + * about it describes the same project: the editor's own parse job + * when the panel is pinned to one, else the deployed version. Sent + * with the selection for the same reason the buffer is — it must not + * be able to disagree with the node on screen. */ + pin: DbtGraphPin, + /** Column lineage the CONSUMERS of this project declare — a script + * reading a model's column and writing a ducklake one. It comes off + * the same graph response, and the details pane merges it with the + * project's own so a trace crosses that boundary instead of ending + * at it. */ + producerColumns: ColumnLineageGraph ) => void } = $props() @@ -364,6 +382,25 @@ // graph that actually came back. let editorParsed = $derived(refreshJob != undefined && raw?.dbt_snapshot_job === refreshJob) + // Which stored graph is on screen. Anything the details pane fetches about a + // selected node asks for this one, so it cannot describe a node parsed from + // the buffer with the deployed version's answer. + let pin = $derived( + editorParsed && refreshJob ? { jobId: refreshJob } : { scriptHash: deployedHash } + ) + + // What the scripts around this project declare about its columns. Empty for + // the ordinary project nothing downstream annotates. + // + // A consumer is anchored here only by its `// materialize` target: this graph + // is fetched `asset_kinds=dbt` so the canvas is the project and nothing else, + // and `buildColumnGraph`'s other anchor is a ducklake WRITE EDGE, which that + // filter drops. So a script consuming a model and writing a ducklake table it + // never declared contributes no hop in this editor, while it does on the + // pipeline page, whose graph spans both kinds. Widening the request would put + // ducklake nodes on the dbt canvas, which is the opposite of what it is for. + let producerColumns = $derived(graph ? buildColumnGraph(graph) : EMPTY_COLUMN_GRAPH) + // `untrack`, because the effect that reloads the graph clears the selection // through here: reading the graph to describe a selection would subscribe that // effect to the very state its own fetch writes, and it would reload forever. @@ -374,7 +411,9 @@ sel?.kind === 'asset' ? graph?.assets.find((a) => a.kind === sel.asset_kind && a.path === sel.path)?.dbt : undefined, - editorParsed ? parsedBuffer : undefined + editorParsed ? parsedBuffer : undefined, + pin, + producerColumns ) ) } @@ -405,7 +444,6 @@ if (deployedHash != undefined) return 'as of last deploy' return 'never parsed' }) -
@@ -435,8 +473,8 @@ {#if refreshPending}
- Still parsing. A cold worker provisions the dbt engine before it starts; a project - pinned to a worker tag nothing serves waits here indefinitely. + Still parsing. A cold worker provisions the dbt engine before it starts; a project pinned to a + worker tag nothing serves waits here indefinitely. { + const sel = pe.selection + if (pe.activeDraft || sel?.kind !== 'asset') return [] + if (sel.asset_kind === 'dbt') return [sel.path] + const seeds = assetColumnNodes(producerColumnGraph, sel.asset_kind, sel.path) + const paths = new Set() + for (const id of connectedComponent(seeds, producerColumnGraph)) { + const node = producerColumnGraph.nodes.get(id) + if (node?.kind === 'dbt') paths.add(node.path) + } + return [...paths] + }) + const dbtColumnLineage = useDbtColumnLineage({ + workspace: () => $workspaceStore, + assetPaths: () => dbtSeedPaths + }) + // One graph across both, so a trace crosses the dbt/ducklake boundary in + // either direction rather than stopping at it. + let columnGraph = $derived(mergeColumnGraphs(producerColumnGraph, dbtColumnLineage.graph)) // Producer-side facts for the editor's live schema-contract diagnostics: // which assets are muted (`on_schema_change=ignore`) and which `_current` @@ -2602,6 +2628,8 @@ {selectionProducers} {selectionDbt} selectionColumnGraph={pe.activeDraft ? EMPTY_COLUMN_GRAPH : columnGraph} + selectionColumnLoading={dbtColumnLineage.loading} + selectionColumnTruncated={dbtColumnLineage.truncated} {schemaCanEvolve} {selectionForkMaterialization} {schemaContractContext} From d3f305db982b7c5dc49babf9bec8b62adcd2557d Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 8 Sep 2026 09:37:56 +0000 Subject: [PATCH 020/144] feat: retry a workflow-as-code task from its task options (#11013) * feat: retry a workflow-as-code task from its task options Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01WvjgKMRNtRNPnAkg6MkiTA * fix: claim every retry attempt key up front, so a step cannot alias one Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01WvjgKMRNtRNPnAkg6MkiTA * fix: bound retry attempts, which now claim their keys up front Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01WvjgKMRNtRNPnAkg6MkiTA * fix: honour an explicit zero retry multiplier in the python client Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01WvjgKMRNtRNPnAkg6MkiTA * docs: state the retry validation rules once in the task docstring Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01WvjgKMRNtRNPnAkg6MkiTA --------- Co-authored-by: Claude Opus 5 (1M context) --- cli/src/guidance/skills.gen.ts | 91 ++++++- docs/wac-sdk-e2e.md | 8 +- python-client/wmill/tests/test_workflow.py | 211 ++++++++++++++++ python-client/wmill/wmill/client.py | 171 +++++++++++-- system_prompts/auto-generated/prompts.ts | 91 ++++++- system_prompts/auto-generated/script.md | 29 ++- system_prompts/auto-generated/sdks/python.md | 29 ++- .../auto-generated/sdks/wac-python.md | 29 ++- .../auto-generated/sdks/wac-typescript.md | 33 ++- .../skills/write-script-python3/SKILL.md | 29 ++- .../skills/write-workflow-as-code/SKILL.md | 62 ++++- system_prompts/generate.py | 7 +- typescript-client/build.sh | 2 +- typescript-client/client.ts | 215 +++++++++++++---- .../tests/workflow_retry.test.ts | 225 ++++++++++++++++++ 15 files changed, 1126 insertions(+), 106 deletions(-) create mode 100644 typescript-client/tests/workflow_retry.test.ts diff --git a/cli/src/guidance/skills.gen.ts b/cli/src/guidance/skills.gen.ts index 5271167a07..a8790f35a7 100644 --- a/cli/src/guidance/skills.gen.ts +++ b/cli/src/guidance/skills.gen.ts @@ -4580,6 +4580,22 @@ def parse_sql_client_name(name: str) -> tuple[str, Optional[str]] # decoded back before the caller sees it: a \`\`datetime\`\` comes back as a # string, a tuple as a list. # +# \`\`retry\`\` re-dispatches the task after a failure, inside \`\`@workflow\`\` only. +# Every attempt is a step of its own (\`\`call_api\`\`, \`\`call_api#2\`\`, ...) and +# the wait between two of them is a durable sleep, so a retrying task holds no +# worker while it backs off. Keys: \`\`attempts\`\` (retries after the first +# failure, a whole number from 0 to 100), \`\`delay\`\` (seconds before the first +# retry, sub-second delays dropped), \`\`multiplier\`\` (applied to the delay +# after each attempt, 1 keeps it constant), \`\`max_delay\`\` (ceiling in +# seconds). \`\`attempts\`\` is required, and an out-of-range or unknown key is +# rejected where the policy is written. +# +# A workflow sleeps once per round, so tasks backing off in the same fan-out +# wait one after another rather than together: the delay before a fan-out +# retries is the sum of every backoff pending in it, not the longest one, and +# it grows with both the width of the fan-out and \`\`attempts\`\`. Retries with +# no \`\`delay\`\` all go out in a single round. +# # Usage:: # # @task @@ -4587,10 +4603,15 @@ def parse_sql_client_name(name: str) -> tuple[str, Optional[str]] # # @task(path="f/external_script", timeout=600, tag="gpu") # async def run_external(x: int): ... -def task(_func = None, path: Optional[str] = None, tag: Optional[str] = None, timeout: Optional[int] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None) +# +# @task(retry={"attempts": 3, "delay": 30, "multiplier": 2}) +# async def call_api(payload: dict): ... +def task(_func = None, path: Optional[str] = None, tag: Optional[str] = None, timeout: Optional[int] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None) # Create a task that dispatches to a separate Windmill script. # +# \`\`retry\`\` takes the same policy as :func:\`task\`. +# # Usage:: # # extract = task_script("f/data/extract", timeout=600) @@ -4598,10 +4619,12 @@ def task(_func = None, path: Optional[str] = None, tag: Optional[str] = None, ti # @workflow # async def main(): # data = await extract(url="https://...") -def task_script(path: str, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None) +def task_script(path: str, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None) # Create a task that dispatches to a separate Windmill flow. # +# \`\`retry\`\` takes the same policy as :func:\`task\`. +# # Usage:: # # pipeline = task_flow("f/etl/pipeline", priority=10) @@ -4609,7 +4632,7 @@ def task_script(path: str, timeout: Optional[int] = None, tag: Optional[str] = N # @workflow # async def main(): # result = await pipeline(input=data) -def task_flow(path: str, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None) +def task_flow(path: str, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None) # Decorator marking an async function as a workflow-as-code entry point. # @@ -6657,6 +6680,34 @@ A caught failure reads the same whether it came from a task or from a \`step()\` Import: \`import { workflow, task, taskScript, taskFlow, step, sleep, waitForApproval, getApprovalUrls, getResumeUrls, parallel } from "windmill-client"\` \`\`\`typescript +/** + * Re-dispatch policy for a failed task. + * + * Every attempt is a step of its own (\`fetch\`, \`fetch#2\`, \`fetch#3\`), and the + * wait between two of them is a durable sleep, so a retrying task holds no + * worker while it backs off. + * + * A workflow sleeps once per round, so tasks backing off in the same fan-out + * wait one after another rather than together: the delay before a fan-out + * retries is the sum of every backoff pending in it, not the longest one, and + * it grows with both the width of the fan-out and \`attempts\`. Retries with no + * \`delay\` all go out in a single round. + */ +export interface TaskRetry { + /** Attempts after the first failure: \`2\` runs the task at most 3 times. + * A whole number from 0 to 100; anything else is rejected where the policy + * is written. */ + attempts: number; + /** Seconds to wait before the first retry. Default 0, retry immediately. + * Sub-second delays are dropped — a durable sleep resolves to the second. */ + delay?: number; + /** Applied to the delay after each attempt: 1 (the default) keeps it + * constant, 2 doubles it. */ + multiplier?: number; + /** Ceiling for the delay in seconds, for a \`multiplier\` above 1. */ + max_delay?: number; +} + export interface TaskOptions { timeout?: number; tag?: string; @@ -6665,6 +6716,7 @@ export interface TaskOptions { concurrency_limit?: number; concurrency_key?: string; concurrency_time_window_s?: number; + retry?: TaskRetry; } /** @@ -6682,9 +6734,11 @@ export async function getResumeUrls(approver?: string, flowLevel?: boolean): Pro * @example * const extract_data = task(async (url: string) => { ... }); * const run_external = task("f/external_script", async (x: number) => { ... }); + * const call_api = task(fetchOrders, { retry: { attempts: 3, delay: 30, multiplier: 2 } }); * * Inside a \`workflow()\`, calling a task dispatches it as a step. - * Outside a workflow, the function body executes directly. + * Outside a workflow, the function body executes directly and + * {@link TaskOptions} — retry included — does not apply. * * A task runs as its own job, so its result is always encoded as JSON and * decoded back before the caller sees it: a \`Date\` comes back as a string, a @@ -6828,6 +6882,22 @@ def get_resume_urls(approver: str = None, flow_level: bool = None) -> dict # decoded back before the caller sees it: a \`\`datetime\`\` comes back as a # string, a tuple as a list. # +# \`\`retry\`\` re-dispatches the task after a failure, inside \`\`@workflow\`\` only. +# Every attempt is a step of its own (\`\`call_api\`\`, \`\`call_api#2\`\`, ...) and +# the wait between two of them is a durable sleep, so a retrying task holds no +# worker while it backs off. Keys: \`\`attempts\`\` (retries after the first +# failure, a whole number from 0 to 100), \`\`delay\`\` (seconds before the first +# retry, sub-second delays dropped), \`\`multiplier\`\` (applied to the delay +# after each attempt, 1 keeps it constant), \`\`max_delay\`\` (ceiling in +# seconds). \`\`attempts\`\` is required, and an out-of-range or unknown key is +# rejected where the policy is written. +# +# A workflow sleeps once per round, so tasks backing off in the same fan-out +# wait one after another rather than together: the delay before a fan-out +# retries is the sum of every backoff pending in it, not the longest one, and +# it grows with both the width of the fan-out and \`\`attempts\`\`. Retries with +# no \`\`delay\`\` all go out in a single round. +# # Usage:: # # @task @@ -6835,10 +6905,15 @@ def get_resume_urls(approver: str = None, flow_level: bool = None) -> dict # # @task(path="f/external_script", timeout=600, tag="gpu") # async def run_external(x: int): ... -def task(_func = None, *, path: Optional[str] = None, tag: Optional[str] = None, timeout: Optional[int] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None) +# +# @task(retry={"attempts": 3, "delay": 30, "multiplier": 2}) +# async def call_api(payload: dict): ... +def task(_func = None, *, path: Optional[str] = None, tag: Optional[str] = None, timeout: Optional[int] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None) # Create a task that dispatches to a separate Windmill script. # +# \`\`retry\`\` takes the same policy as :func:\`task\`. +# # Usage:: # # extract = task_script("f/data/extract", timeout=600) @@ -6846,10 +6921,12 @@ def task(_func = None, *, path: Optional[str] = None, tag: Optional[str] = None, # @workflow # async def main(): # data = await extract(url="https://...") -def task_script(path: str, *, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None) +def task_script(path: str, *, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None) # Create a task that dispatches to a separate Windmill flow. # +# \`\`retry\`\` takes the same policy as :func:\`task\`. +# # Usage:: # # pipeline = task_flow("f/etl/pipeline", priority=10) @@ -6857,7 +6934,7 @@ def task_script(path: str, *, timeout: Optional[int] = None, tag: Optional[str] # @workflow # async def main(): # result = await pipeline(input=data) -def task_flow(path: str, *, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None) +def task_flow(path: str, *, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None) # Decorator marking an async function as a workflow-as-code entry point. # diff --git a/docs/wac-sdk-e2e.md b/docs/wac-sdk-e2e.md index 5c6682d93d..0920bdf168 100644 --- a/docs/wac-sdk-e2e.md +++ b/docs/wac-sdk-e2e.md @@ -34,8 +34,12 @@ cp python-client/wmill/wmill/client.py \ /tmp/windmill-mytest/cache/python_3_12/wmill==*/wmill/client.py find /tmp/windmill-mytest -name __pycache__ -type d -exec rm -rf {} + -# 4. RESTART the backend — see below -# 5. run your scenarios, and rm -rf /tmp/windmill-mytest when done +# 4. drop the bundle snapshots — a bun job runs a bundle built from the package, +# cached by content hash, so a job that already ran keeps the old SDK inlined +rm -rf /tmp/windmill-mytest/cache/bun + +# 5. RESTART the backend — see below +# 6. run your scenarios, and rm -rf /tmp/windmill-mytest when done ``` ## Restart the workers after injecting diff --git a/python-client/wmill/tests/test_workflow.py b/python-client/wmill/tests/test_workflow.py index f0b9d4b801..ceed39bfeb 100644 --- a/python-client/wmill/tests/test_workflow.py +++ b/python-client/wmill/tests/test_workflow.py @@ -1436,6 +1436,217 @@ class TestTaskOptions: assert step_info["concurrency_time_window_s"] == 60 +# ===================================================================== +# TASK RETRY TESTS +# ===================================================================== + + +# What the failed child job leaves in ``completed_steps``. +_FAILED = {"__wmill_error": True, "message": "boom", "result": {}} + + +class TestTaskRetry: + """Nothing carries a retry across rounds: every round re-derives which + attempt comes next from the checkpoint alone.""" + + def test_each_failure_buys_a_backoff_sleep_and_one_more_attempt(self): + @task(retry={"attempts": 2, "delay": 30, "multiplier": 2}) + async def call_api(x: int): + return x + + @workflow + async def wf(x: int): + return await call_api(x=x) + + completed = {"call_api": _FAILED} + r = _run_workflow(wf, {"completed_steps": dict(completed)}, {"x": 1}) + assert r == {"type": "sleep", "key": "call_api#retry2", "seconds": 30} + + completed["call_api#retry2"] = None + r = _run_workflow(wf, {"completed_steps": dict(completed)}, {"x": 1}) + assert [s["key"] for s in r["steps"]] == ["call_api#2"] + + # the delay grows by the multiplier for the second retry + completed["call_api#2"] = _FAILED + r = _run_workflow(wf, {"completed_steps": dict(completed)}, {"x": 1}) + assert r == {"type": "sleep", "key": "call_api#retry3", "seconds": 60} + + completed["call_api#retry3"] = None + r = _run_workflow(wf, {"completed_steps": dict(completed)}, {"x": 1}) + assert [s["key"] for s in r["steps"]] == ["call_api#3"] + + # attempts spent: the failure reaches the body + completed["call_api#3"] = _FAILED + with pytest.raises(TaskError): + _run_workflow(wf, {"completed_steps": dict(completed)}, {"x": 1}) + + def test_a_task_the_body_never_awaits_still_sleeps_and_retries(self): + # The runner dispatches unawaited task calls by flushing ``_pending``, so + # a backoff that only fired when awaited would drop the retry silently + # and report the workflow complete. + @task(retry={"attempts": 1, "delay": 30}) + async def fire(x: int): + return x + + @workflow + async def wf(): + fire(x=1) + return "done" + + r = _run_workflow(wf, {"completed_steps": {"fire": _FAILED}}, {}) + assert r == {"type": "sleep", "key": "fire#retry2", "seconds": 30} + + # ``step()`` names are arbitrary strings, so a step really can be called + # ``t#2``. Whichever of the two allocates second is the one renamed, and it + # has to be the same one in every round — hence claiming the attempt keys up + # front. + def test_an_inline_step_named_like_an_attempt_key_before_the_task_keeps_it(self): + @task(retry={"attempts": 1}) + async def t(x: int): + return x + + @workflow + async def wf(): + decoy = await step("t#2", lambda: "not an attempt") + return [decoy, await t(x=1)] + + r = _run_workflow( + wf, {"completed_steps": {"t#2": "not an attempt", "t": _FAILED}}, {} + ) + assert [s["key"] for s in r["steps"]] == ["t#2_2"] + + def test_an_inline_step_named_like_an_attempt_key_after_the_task_is_not_it(self): + @task(retry={"attempts": 1}) + async def t(x: int): + return x + + @workflow + async def wf(): + pending = t(x=1) + decoy = await step("t#2", lambda: "not an attempt") + return [decoy, await pending] + + # The first round records the step under the key left over after the + # task claimed ``t#2``, so the retry re-dispatches instead of reading + # the step's value. + r = _run_workflow(wf, {}, {}) + assert r["key"] == "t#2_2" + + r = _run_workflow( + wf, {"completed_steps": {"t": _FAILED, "t#2_2": "not an attempt"}}, {} + ) + assert [s["key"] for s in r["steps"]] == ["t#2"] + + def test_the_child_dispatched_for_an_attempt_walks_past_failure_and_backoff(self): + # The non-matching branch awaits a future that never resolves, so a child + # that walks the loop wrong parks the run until its timeout. + @task(retry={"attempts": 1, "delay": 30}) + async def t(x: int): + return x * 10 + + @workflow + async def wf(x: int): + return await t(x=x) + + r = _run_workflow( + wf, + { + "completed_steps": {"t": _FAILED, "t#retry2": None}, + "_executing_key": "t#2", + }, + {"x": 4}, + ) + assert r == {"type": "complete", "result": 40} + + def test_a_misspelled_retry_option_raises_instead_of_being_ignored(self): + # The policy is a plain dict here, unlike the TS `TaskRetry` type, so + # nothing else would tell the author the option never took effect. + with pytest.raises(ValueError, match="max_delay_s"): + + @task(retry={"attempts": 2, "max_delay_s": 300}) + async def t(x: int): + return x + + def test_an_out_of_range_attempt_count_is_rejected_where_it_is_written(self): + # Each attempt claims its keys before the first one is dispatched, so an + # unbounded count would hang the workflow allocating them. + for attempts in (10_000, -1, 2.5, float("inf")): + with pytest.raises(ValueError, match="whole number"): + + @task(retry={"attempts": attempts}) + async def t(x: int): + return x + + def test_a_zero_multiplier_is_honoured_not_read_as_the_default_of_one(self): + @task(retry={"attempts": 2, "delay": 30, "multiplier": 0}) + async def t(x: int): + return x + + @workflow + async def wf(x: int): + return await t(x=x) + + r = _run_workflow(wf, {"completed_steps": {"t": _FAILED}}, {"x": 1}) + assert r == {"type": "sleep", "key": "t#retry2", "seconds": 30} + + # 30 * 0 — the second retry goes out with no wait at all + r = _run_workflow( + wf, + {"completed_steps": {"t": _FAILED, "t#retry2": None, "t#2": _FAILED}}, + {"x": 1}, + ) + assert [s["key"] for s in r["steps"]] == ["t#3"] + + def test_max_delay_caps_the_backoff_a_multiplier_grows(self): + @task(retry={"attempts": 2, "delay": 60, "multiplier": 100, "max_delay": 300}) + async def t(x: int): + return x + + @workflow + async def wf(x: int): + return await t(x=x) + + r = _run_workflow( + wf, + {"completed_steps": {"t": _FAILED, "t#retry2": None, "t#2": _FAILED}}, + {"x": 1}, + ) + assert r == {"type": "sleep", "key": "t#retry3", "seconds": 300} + + def test_retry_that_succeeds_resolves_and_later_steps_keep_their_keys(self): + # No delay, so the retry is dispatched without a sleep round in between. + @task(retry={"attempts": 1}) + async def flaky(x: int): + return x + + @workflow + async def wf(x: int): + return await double(x=await flaky(x=x)) + + r = _run_workflow( + wf, {"completed_steps": {"flaky": _FAILED, "flaky#2": 7}}, {"x": 1} + ) + assert r["steps"][0]["key"] == "double" + assert r["steps"][0]["args"] == {"x": 7} + + def test_retrying_one_call_does_not_move_the_keys_of_the_calls_beside_it(self): + @task(retry={"attempts": 1}) + async def t(x: int): + return x + + @workflow + async def wf(): + return await asyncio.gather(t(x=1), t(x=2)) + + r = _run_workflow(wf, {}, {}) + assert [s["key"] for s in r["steps"]] == ["t", "t_2"] + + # the first call retries as ``t#2``; the second keeps the ``t_2`` it was + # dispatched under, rather than being read as the first call's retry + r = _run_workflow(wf, {"completed_steps": {"t": _FAILED, "t_2": 20}}, {}) + assert [s["key"] for s in r["steps"]] == ["t#2"] + + # ===================================================================== # SLEEP TESTS # ===================================================================== diff --git a/python-client/wmill/wmill/client.py b/python-client/wmill/wmill/client.py index 3c0ce050fb..98bbd7c9f3 100644 --- a/python-client/wmill/wmill/client.py +++ b/python-client/wmill/wmill/client.py @@ -2874,6 +2874,59 @@ def _task_error_from_marker(marker: dict, fallback_message: str) -> TaskError: ) +# The worker deserializes a sleep into a ``u32`` of seconds and fails the whole +# job on anything wider, so a delay a multiplier has run away with has to be +# capped here rather than sent. +_MAX_SLEEP_SECONDS = 2**32 - 1 + +_RETRY_KEYS = ("attempts", "delay", "multiplier", "max_delay") + +# Every attempt claims its keys before the first one is dispatched, so an +# unbounded ``attempts`` is a workflow that hangs allocating rather than a very +# patient one. +_MAX_RETRY_ATTEMPTS = 100 + + +def _checked_retry(retry: Optional[dict]) -> Optional[dict]: + """Reject a policy where it is written, rather than mid-run on a replay: the + policy is a plain dict, so a misspelled key would otherwise be dropped in + silence and the task would retry on a policy nobody wrote.""" + if retry is None: + return None + unknown = sorted(k for k in retry if k not in _RETRY_KEYS) + if unknown: + raise ValueError( + f"unknown retry option(s): {', '.join(unknown)}. Expected any of: {', '.join(_RETRY_KEYS)}" + ) + attempts = retry.get("attempts") + if isinstance(attempts, bool) or not isinstance(attempts, int) or not 0 <= attempts <= _MAX_RETRY_ATTEMPTS: + raise ValueError( + f"retry attempts must be a whole number between 0 and {_MAX_RETRY_ATTEMPTS}, got {attempts!r}" + ) + return retry + + +def _retry_delay_seconds(retry: dict, attempt: int) -> int: + """Seconds to wait before retry number ``attempt`` (0 is the first retry).""" + base = retry.get("delay") or 0 + if base <= 0: + return 0 + # `or 1` would read an explicit `multiplier: 0` — every retry after the + # first going out with no wait — as the default of 1. + multiplier = retry.get("multiplier") + if multiplier is None: + multiplier = 1 + try: + grown = base * multiplier**attempt + except OverflowError: + # A float delay times an integer multiplier raised past ~1e308. + grown = _MAX_SLEEP_SECONDS + max_delay = retry.get("max_delay") + if max_delay is not None: + grown = min(grown, max_delay) + return max(0, int(min(grown, _MAX_SLEEP_SECONDS))) + + _workflow_ctx: _contextvars.ContextVar["WorkflowCtx"] = _contextvars.ContextVar( "_workflow_ctx" ) @@ -2929,28 +2982,77 @@ class WorkflowCtx: def _next_step(self, name: str, script: str, func=None, dispatch_type: str = "inline", _task_options: Optional[dict] = None, **kwargs): """Return an awaitable that either resolves from cache or suspends.""" - key = self._alloc_key(name or script or "step") + step_name = name or script or "step" + retry = (_task_options or {}).get("retry") or {} + # Clamped as well as validated at decoration: a policy that reached here + # another way must not spin the key loop below. + max_retries = min(max(0, int(retry.get("attempts") or 0)), _MAX_RETRY_ATTEMPTS) + # Claimed up front, all of them, and named off the first attempt's key: + # one allocated later would shift the keys of the steps beside it, and a + # ``step()`` named ``t#2`` — names are arbitrary — could alias one. + # Whichever is allocated second is the one renamed, in every round alike. + base_key = self._alloc_key(step_name) + attempt_keys = [base_key] + backoff_keys = [] + for i in range(max_retries): + backoff_keys.append(self._alloc_key(f"{base_key}#retry{i + 2}")) + attempt_keys.append(self._alloc_key(f"{base_key}#{i + 2}")) + + # One pass per attempt. Every attempt the checkpoint already holds is + # decided here — a failed one either retries (moving to the next key) or + # is handed back to the body — so the loop always ends at the first + # attempt that has yet to run. + attempt = 0 + while True: + key = attempt_keys[attempt] + + if key in self._completed: + val = self._completed[key] + if isinstance(val, dict) and val.get("__wmill_error"): + if attempt < max_retries: + self._retry_backoff(backoff_keys[attempt], base_key, retry, attempt) + attempt += 1 + continue + raise _task_error_from_marker(val, f"Task '{name}' failed") + return self._resolved(val) + + if self._executing_key is not None: + if key == self._executing_key: + return self._execute_directly(func, **kwargs) + else: + return self._never_resolve() + + print(f"\n--- WAC: {key} ---") + info = {"name": name or key, "script": script or key, "args": kwargs, "key": key, "dispatch_type": dispatch_type} + if _task_options: + for opt_key in ("timeout", "tag", "cache_ttl", "priority", "concurrent_limit", "concurrency_key", "concurrency_time_window_s"): + if opt_key in _task_options and _task_options[opt_key] is not None: + info[opt_key] = _task_options[opt_key] + self._pending.append(info) + return self._suspend() + + def _retry_backoff(self, key: str, base_key: str, retry: dict, attempt: int) -> None: + """Wait out the backoff between two attempts of a retried task, as a + durable sleep, and return once there is nothing to wait for — no delay + configured, or the sleep already in the checkpoint. + + Raises where it stands rather than from a coroutine the caller has to + await: a task call the body never awaits is still dispatched (the runner + flushes ``_pending``), so a backoff that only fired when awaited would + drop the retry and let the round report the workflow complete.""" + seconds = _retry_delay_seconds(retry, attempt) + if seconds < 1: + return if key in self._completed: - val = self._completed[key] - if isinstance(val, dict) and val.get("__wmill_error"): - raise _task_error_from_marker(val, f"Task '{name}' failed") - return self._resolved(val) - + return + # Child mode never raises: the parent dispatched this child only after + # its own round had slept, so the loop moves on to the attempt being + # executed. if self._executing_key is not None: - if key == self._executing_key: - return self._execute_directly(func, **kwargs) - else: - return self._never_resolve() - - print(f"\n--- WAC: {key} ---") - info = {"name": name or key, "script": script or key, "args": kwargs, "key": key, "dispatch_type": dispatch_type} - if _task_options: - for opt_key in ("timeout", "tag", "cache_ttl", "priority", "concurrent_limit", "concurrency_key", "concurrency_time_window_s"): - if opt_key in _task_options and _task_options[opt_key] is not None: - info[opt_key] = _task_options[opt_key] - self._pending.append(info) - return self._suspend() + return + print(f"\n--- WAC: sleep({key}, {seconds}s) before retrying {base_key} ---") + raise _StepSuspend({"mode": "sleep", "key": key, "seconds": seconds, "steps": []}) async def _resolved(self, value): return value @@ -3190,6 +3292,7 @@ def task( concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, + retry: Optional[dict] = None, ): """Decorator that marks a function as a workflow task. @@ -3204,6 +3307,22 @@ def task( decoded back before the caller sees it: a ``datetime`` comes back as a string, a tuple as a list. + ``retry`` re-dispatches the task after a failure, inside ``@workflow`` only. + Every attempt is a step of its own (``call_api``, ``call_api#2``, ...) and + the wait between two of them is a durable sleep, so a retrying task holds no + worker while it backs off. Keys: ``attempts`` (retries after the first + failure, a whole number from 0 to 100), ``delay`` (seconds before the first + retry, sub-second delays dropped), ``multiplier`` (applied to the delay + after each attempt, 1 keeps it constant), ``max_delay`` (ceiling in + seconds). ``attempts`` is required, and an out-of-range or unknown key is + rejected where the policy is written. + + A workflow sleeps once per round, so tasks backing off in the same fan-out + wait one after another rather than together: the delay before a fan-out + retries is the sum of every backoff pending in it, not the longest one, and + it grows with both the width of the fan-out and ``attempts``. Retries with + no ``delay`` all go out in a single round. + Usage:: @task @@ -3211,6 +3330,9 @@ def task( @task(path="f/external_script", timeout=600, tag="gpu") async def run_external(x: int): ... + + @task(retry={"attempts": 3, "delay": 30, "multiplier": 2}) + async def call_api(payload: dict): ... """ from inspect import signature as _sig @@ -3222,6 +3344,7 @@ def task( "concurrent_limit": concurrency_limit, "concurrency_key": concurrency_key, "concurrency_time_window_s": concurrency_time_window_s, + "retry": _checked_retry(retry), } # Remove None values _task_opts = {k: v for k, v in _task_opts.items() if v is not None} or None @@ -3316,9 +3439,12 @@ def task_script( concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, + retry: Optional[dict] = None, ): """Create a task that dispatches to a separate Windmill script. + ``retry`` takes the same policy as :func:`task`. + Usage:: extract = task_script("f/data/extract", timeout=600) @@ -3328,7 +3454,7 @@ def task_script( data = await extract(url="https://...") """ name = path.rsplit("/", 1)[-1] - _opts = {k: v for k, v in {"timeout": timeout, "tag": tag, "cache_ttl": cache_ttl, "priority": priority, "concurrent_limit": concurrency_limit, "concurrency_key": concurrency_key, "concurrency_time_window_s": concurrency_time_window_s}.items() if v is not None} or None + _opts = {k: v for k, v in {"timeout": timeout, "tag": tag, "cache_ttl": cache_ttl, "priority": priority, "concurrent_limit": concurrency_limit, "concurrency_key": concurrency_key, "concurrency_time_window_s": concurrency_time_window_s, "retry": _checked_retry(retry)}.items() if v is not None} or None def wrapper(**kwargs): ctx = _workflow_ctx.get(None) @@ -3352,9 +3478,12 @@ def task_flow( concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, + retry: Optional[dict] = None, ): """Create a task that dispatches to a separate Windmill flow. + ``retry`` takes the same policy as :func:`task`. + Usage:: pipeline = task_flow("f/etl/pipeline", priority=10) @@ -3364,7 +3493,7 @@ def task_flow( result = await pipeline(input=data) """ name = path.rsplit("/", 1)[-1] - _opts = {k: v for k, v in {"timeout": timeout, "tag": tag, "cache_ttl": cache_ttl, "priority": priority, "concurrent_limit": concurrency_limit, "concurrency_key": concurrency_key, "concurrency_time_window_s": concurrency_time_window_s}.items() if v is not None} or None + _opts = {k: v for k, v in {"timeout": timeout, "tag": tag, "cache_ttl": cache_ttl, "priority": priority, "concurrent_limit": concurrency_limit, "concurrency_key": concurrency_key, "concurrency_time_window_s": concurrency_time_window_s, "retry": _checked_retry(retry)}.items() if v is not None} or None def wrapper(**kwargs): ctx = _workflow_ctx.get(None) diff --git a/system_prompts/auto-generated/prompts.ts b/system_prompts/auto-generated/prompts.ts index 5776fd6d0d..c4b2f5f6cf 100644 --- a/system_prompts/auto-generated/prompts.ts +++ b/system_prompts/auto-generated/prompts.ts @@ -2553,6 +2553,22 @@ def parse_sql_client_name(name: str) -> tuple[str, Optional[str]] # decoded back before the caller sees it: a \`\`datetime\`\` comes back as a # string, a tuple as a list. # +# \`\`retry\`\` re-dispatches the task after a failure, inside \`\`@workflow\`\` only. +# Every attempt is a step of its own (\`\`call_api\`\`, \`\`call_api#2\`\`, ...) and +# the wait between two of them is a durable sleep, so a retrying task holds no +# worker while it backs off. Keys: \`\`attempts\`\` (retries after the first +# failure, a whole number from 0 to 100), \`\`delay\`\` (seconds before the first +# retry, sub-second delays dropped), \`\`multiplier\`\` (applied to the delay +# after each attempt, 1 keeps it constant), \`\`max_delay\`\` (ceiling in +# seconds). \`\`attempts\`\` is required, and an out-of-range or unknown key is +# rejected where the policy is written. +# +# A workflow sleeps once per round, so tasks backing off in the same fan-out +# wait one after another rather than together: the delay before a fan-out +# retries is the sum of every backoff pending in it, not the longest one, and +# it grows with both the width of the fan-out and \`\`attempts\`\`. Retries with +# no \`\`delay\`\` all go out in a single round. +# # Usage:: # # @task @@ -2560,10 +2576,15 @@ def parse_sql_client_name(name: str) -> tuple[str, Optional[str]] # # @task(path="f/external_script", timeout=600, tag="gpu") # async def run_external(x: int): ... -def task(_func = None, path: Optional[str] = None, tag: Optional[str] = None, timeout: Optional[int] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None) +# +# @task(retry={"attempts": 3, "delay": 30, "multiplier": 2}) +# async def call_api(payload: dict): ... +def task(_func = None, path: Optional[str] = None, tag: Optional[str] = None, timeout: Optional[int] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None) # Create a task that dispatches to a separate Windmill script. # +# \`\`retry\`\` takes the same policy as :func:\`task\`. +# # Usage:: # # extract = task_script("f/data/extract", timeout=600) @@ -2571,10 +2592,12 @@ def task(_func = None, path: Optional[str] = None, tag: Optional[str] = None, ti # @workflow # async def main(): # data = await extract(url="https://...") -def task_script(path: str, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None) +def task_script(path: str, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None) # Create a task that dispatches to a separate Windmill flow. # +# \`\`retry\`\` takes the same policy as :func:\`task\`. +# # Usage:: # # pipeline = task_flow("f/etl/pipeline", priority=10) @@ -2582,7 +2605,7 @@ def task_script(path: str, timeout: Optional[int] = None, tag: Optional[str] = N # @workflow # async def main(): # result = await pipeline(input=data) -def task_flow(path: str, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None) +def task_flow(path: str, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None) # Decorator marking an async function as a workflow-as-code entry point. # @@ -2661,6 +2684,34 @@ export const WAC_SDK_TYPESCRIPT = `## TypeScript Workflow-as-Code API (windmill- Import: \`import { workflow, task, taskScript, taskFlow, step, sleep, waitForApproval, getApprovalUrls, getResumeUrls, parallel } from "windmill-client"\` \`\`\`typescript +/** + * Re-dispatch policy for a failed task. + * + * Every attempt is a step of its own (\`fetch\`, \`fetch#2\`, \`fetch#3\`), and the + * wait between two of them is a durable sleep, so a retrying task holds no + * worker while it backs off. + * + * A workflow sleeps once per round, so tasks backing off in the same fan-out + * wait one after another rather than together: the delay before a fan-out + * retries is the sum of every backoff pending in it, not the longest one, and + * it grows with both the width of the fan-out and \`attempts\`. Retries with no + * \`delay\` all go out in a single round. + */ +export interface TaskRetry { + /** Attempts after the first failure: \`2\` runs the task at most 3 times. + * A whole number from 0 to 100; anything else is rejected where the policy + * is written. */ + attempts: number; + /** Seconds to wait before the first retry. Default 0, retry immediately. + * Sub-second delays are dropped — a durable sleep resolves to the second. */ + delay?: number; + /** Applied to the delay after each attempt: 1 (the default) keeps it + * constant, 2 doubles it. */ + multiplier?: number; + /** Ceiling for the delay in seconds, for a \`multiplier\` above 1. */ + max_delay?: number; +} + export interface TaskOptions { timeout?: number; tag?: string; @@ -2669,6 +2720,7 @@ export interface TaskOptions { concurrency_limit?: number; concurrency_key?: string; concurrency_time_window_s?: number; + retry?: TaskRetry; } /** @@ -2686,9 +2738,11 @@ export async function getResumeUrls(approver?: string, flowLevel?: boolean): Pro * @example * const extract_data = task(async (url: string) => { ... }); * const run_external = task("f/external_script", async (x: number) => { ... }); + * const call_api = task(fetchOrders, { retry: { attempts: 3, delay: 30, multiplier: 2 } }); * * Inside a \`workflow()\`, calling a task dispatches it as a step. - * Outside a workflow, the function body executes directly. + * Outside a workflow, the function body executes directly and + * {@link TaskOptions} — retry included — does not apply. * * A task runs as its own job, so its result is always encoded as JSON and * decoded back before the caller sees it: a \`Date\` comes back as a string, a @@ -2832,6 +2886,22 @@ def get_resume_urls(approver: str = None, flow_level: bool = None) -> dict # decoded back before the caller sees it: a \`\`datetime\`\` comes back as a # string, a tuple as a list. # +# \`\`retry\`\` re-dispatches the task after a failure, inside \`\`@workflow\`\` only. +# Every attempt is a step of its own (\`\`call_api\`\`, \`\`call_api#2\`\`, ...) and +# the wait between two of them is a durable sleep, so a retrying task holds no +# worker while it backs off. Keys: \`\`attempts\`\` (retries after the first +# failure, a whole number from 0 to 100), \`\`delay\`\` (seconds before the first +# retry, sub-second delays dropped), \`\`multiplier\`\` (applied to the delay +# after each attempt, 1 keeps it constant), \`\`max_delay\`\` (ceiling in +# seconds). \`\`attempts\`\` is required, and an out-of-range or unknown key is +# rejected where the policy is written. +# +# A workflow sleeps once per round, so tasks backing off in the same fan-out +# wait one after another rather than together: the delay before a fan-out +# retries is the sum of every backoff pending in it, not the longest one, and +# it grows with both the width of the fan-out and \`\`attempts\`\`. Retries with +# no \`\`delay\`\` all go out in a single round. +# # Usage:: # # @task @@ -2839,10 +2909,15 @@ def get_resume_urls(approver: str = None, flow_level: bool = None) -> dict # # @task(path="f/external_script", timeout=600, tag="gpu") # async def run_external(x: int): ... -def task(_func = None, *, path: Optional[str] = None, tag: Optional[str] = None, timeout: Optional[int] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None) +# +# @task(retry={"attempts": 3, "delay": 30, "multiplier": 2}) +# async def call_api(payload: dict): ... +def task(_func = None, *, path: Optional[str] = None, tag: Optional[str] = None, timeout: Optional[int] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None) # Create a task that dispatches to a separate Windmill script. # +# \`\`retry\`\` takes the same policy as :func:\`task\`. +# # Usage:: # # extract = task_script("f/data/extract", timeout=600) @@ -2850,10 +2925,12 @@ def task(_func = None, *, path: Optional[str] = None, tag: Optional[str] = None, # @workflow # async def main(): # data = await extract(url="https://...") -def task_script(path: str, *, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None) +def task_script(path: str, *, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None) # Create a task that dispatches to a separate Windmill flow. # +# \`\`retry\`\` takes the same policy as :func:\`task\`. +# # Usage:: # # pipeline = task_flow("f/etl/pipeline", priority=10) @@ -2861,7 +2938,7 @@ def task_script(path: str, *, timeout: Optional[int] = None, tag: Optional[str] # @workflow # async def main(): # result = await pipeline(input=data) -def task_flow(path: str, *, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None) +def task_flow(path: str, *, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None) # Decorator marking an async function as a workflow-as-code entry point. # diff --git a/system_prompts/auto-generated/script.md b/system_prompts/auto-generated/script.md index 0b67fa0027..4d7c02a90d 100644 --- a/system_prompts/auto-generated/script.md +++ b/system_prompts/auto-generated/script.md @@ -2708,6 +2708,22 @@ def parse_sql_client_name(name: str) -> tuple[str, Optional[str]] # decoded back before the caller sees it: a ``datetime`` comes back as a # string, a tuple as a list. # +# ``retry`` re-dispatches the task after a failure, inside ``@workflow`` only. +# Every attempt is a step of its own (``call_api``, ``call_api#2``, ...) and +# the wait between two of them is a durable sleep, so a retrying task holds no +# worker while it backs off. Keys: ``attempts`` (retries after the first +# failure, a whole number from 0 to 100), ``delay`` (seconds before the first +# retry, sub-second delays dropped), ``multiplier`` (applied to the delay +# after each attempt, 1 keeps it constant), ``max_delay`` (ceiling in +# seconds). ``attempts`` is required, and an out-of-range or unknown key is +# rejected where the policy is written. +# +# A workflow sleeps once per round, so tasks backing off in the same fan-out +# wait one after another rather than together: the delay before a fan-out +# retries is the sum of every backoff pending in it, not the longest one, and +# it grows with both the width of the fan-out and ``attempts``. Retries with +# no ``delay`` all go out in a single round. +# # Usage:: # # @task @@ -2715,10 +2731,15 @@ def parse_sql_client_name(name: str) -> tuple[str, Optional[str]] # # @task(path="f/external_script", timeout=600, tag="gpu") # async def run_external(x: int): ... -def task(_func = None, path: Optional[str] = None, tag: Optional[str] = None, timeout: Optional[int] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None) +# +# @task(retry={"attempts": 3, "delay": 30, "multiplier": 2}) +# async def call_api(payload: dict): ... +def task(_func = None, path: Optional[str] = None, tag: Optional[str] = None, timeout: Optional[int] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None) # Create a task that dispatches to a separate Windmill script. # +# ``retry`` takes the same policy as :func:`task`. +# # Usage:: # # extract = task_script("f/data/extract", timeout=600) @@ -2726,10 +2747,12 @@ def task(_func = None, path: Optional[str] = None, tag: Optional[str] = None, ti # @workflow # async def main(): # data = await extract(url="https://...") -def task_script(path: str, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None) +def task_script(path: str, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None) # Create a task that dispatches to a separate Windmill flow. # +# ``retry`` takes the same policy as :func:`task`. +# # Usage:: # # pipeline = task_flow("f/etl/pipeline", priority=10) @@ -2737,7 +2760,7 @@ def task_script(path: str, timeout: Optional[int] = None, tag: Optional[str] = N # @workflow # async def main(): # result = await pipeline(input=data) -def task_flow(path: str, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None) +def task_flow(path: str, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None) # Decorator marking an async function as a workflow-as-code entry point. # diff --git a/system_prompts/auto-generated/sdks/python.md b/system_prompts/auto-generated/sdks/python.md index f30f1a0018..a060267f35 100644 --- a/system_prompts/auto-generated/sdks/python.md +++ b/system_prompts/auto-generated/sdks/python.md @@ -656,6 +656,22 @@ def parse_sql_client_name(name: str) -> tuple[str, Optional[str]] # decoded back before the caller sees it: a ``datetime`` comes back as a # string, a tuple as a list. # +# ``retry`` re-dispatches the task after a failure, inside ``@workflow`` only. +# Every attempt is a step of its own (``call_api``, ``call_api#2``, ...) and +# the wait between two of them is a durable sleep, so a retrying task holds no +# worker while it backs off. Keys: ``attempts`` (retries after the first +# failure, a whole number from 0 to 100), ``delay`` (seconds before the first +# retry, sub-second delays dropped), ``multiplier`` (applied to the delay +# after each attempt, 1 keeps it constant), ``max_delay`` (ceiling in +# seconds). ``attempts`` is required, and an out-of-range or unknown key is +# rejected where the policy is written. +# +# A workflow sleeps once per round, so tasks backing off in the same fan-out +# wait one after another rather than together: the delay before a fan-out +# retries is the sum of every backoff pending in it, not the longest one, and +# it grows with both the width of the fan-out and ``attempts``. Retries with +# no ``delay`` all go out in a single round. +# # Usage:: # # @task @@ -663,10 +679,15 @@ def parse_sql_client_name(name: str) -> tuple[str, Optional[str]] # # @task(path="f/external_script", timeout=600, tag="gpu") # async def run_external(x: int): ... -def task(_func = None, path: Optional[str] = None, tag: Optional[str] = None, timeout: Optional[int] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None) +# +# @task(retry={"attempts": 3, "delay": 30, "multiplier": 2}) +# async def call_api(payload: dict): ... +def task(_func = None, path: Optional[str] = None, tag: Optional[str] = None, timeout: Optional[int] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None) # Create a task that dispatches to a separate Windmill script. # +# ``retry`` takes the same policy as :func:`task`. +# # Usage:: # # extract = task_script("f/data/extract", timeout=600) @@ -674,10 +695,12 @@ def task(_func = None, path: Optional[str] = None, tag: Optional[str] = None, ti # @workflow # async def main(): # data = await extract(url="https://...") -def task_script(path: str, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None) +def task_script(path: str, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None) # Create a task that dispatches to a separate Windmill flow. # +# ``retry`` takes the same policy as :func:`task`. +# # Usage:: # # pipeline = task_flow("f/etl/pipeline", priority=10) @@ -685,7 +708,7 @@ def task_script(path: str, timeout: Optional[int] = None, tag: Optional[str] = N # @workflow # async def main(): # result = await pipeline(input=data) -def task_flow(path: str, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None) +def task_flow(path: str, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None) # Decorator marking an async function as a workflow-as-code entry point. # diff --git a/system_prompts/auto-generated/sdks/wac-python.md b/system_prompts/auto-generated/sdks/wac-python.md index e1ac8635f8..e7f92d32d0 100644 --- a/system_prompts/auto-generated/sdks/wac-python.md +++ b/system_prompts/auto-generated/sdks/wac-python.md @@ -42,6 +42,22 @@ def get_resume_urls(approver: str = None, flow_level: bool = None) -> dict # decoded back before the caller sees it: a ``datetime`` comes back as a # string, a tuple as a list. # +# ``retry`` re-dispatches the task after a failure, inside ``@workflow`` only. +# Every attempt is a step of its own (``call_api``, ``call_api#2``, ...) and +# the wait between two of them is a durable sleep, so a retrying task holds no +# worker while it backs off. Keys: ``attempts`` (retries after the first +# failure, a whole number from 0 to 100), ``delay`` (seconds before the first +# retry, sub-second delays dropped), ``multiplier`` (applied to the delay +# after each attempt, 1 keeps it constant), ``max_delay`` (ceiling in +# seconds). ``attempts`` is required, and an out-of-range or unknown key is +# rejected where the policy is written. +# +# A workflow sleeps once per round, so tasks backing off in the same fan-out +# wait one after another rather than together: the delay before a fan-out +# retries is the sum of every backoff pending in it, not the longest one, and +# it grows with both the width of the fan-out and ``attempts``. Retries with +# no ``delay`` all go out in a single round. +# # Usage:: # # @task @@ -49,10 +65,15 @@ def get_resume_urls(approver: str = None, flow_level: bool = None) -> dict # # @task(path="f/external_script", timeout=600, tag="gpu") # async def run_external(x: int): ... -def task(_func = None, *, path: Optional[str] = None, tag: Optional[str] = None, timeout: Optional[int] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None) +# +# @task(retry={"attempts": 3, "delay": 30, "multiplier": 2}) +# async def call_api(payload: dict): ... +def task(_func = None, *, path: Optional[str] = None, tag: Optional[str] = None, timeout: Optional[int] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None) # Create a task that dispatches to a separate Windmill script. # +# ``retry`` takes the same policy as :func:`task`. +# # Usage:: # # extract = task_script("f/data/extract", timeout=600) @@ -60,10 +81,12 @@ def task(_func = None, *, path: Optional[str] = None, tag: Optional[str] = None, # @workflow # async def main(): # data = await extract(url="https://...") -def task_script(path: str, *, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None) +def task_script(path: str, *, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None) # Create a task that dispatches to a separate Windmill flow. # +# ``retry`` takes the same policy as :func:`task`. +# # Usage:: # # pipeline = task_flow("f/etl/pipeline", priority=10) @@ -71,7 +94,7 @@ def task_script(path: str, *, timeout: Optional[int] = None, tag: Optional[str] # @workflow # async def main(): # result = await pipeline(input=data) -def task_flow(path: str, *, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None) +def task_flow(path: str, *, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None) # Decorator marking an async function as a workflow-as-code entry point. # diff --git a/system_prompts/auto-generated/sdks/wac-typescript.md b/system_prompts/auto-generated/sdks/wac-typescript.md index f8a2452942..afaeac369a 100644 --- a/system_prompts/auto-generated/sdks/wac-typescript.md +++ b/system_prompts/auto-generated/sdks/wac-typescript.md @@ -3,6 +3,34 @@ Import: `import { workflow, task, taskScript, taskFlow, step, sleep, waitForApproval, getApprovalUrls, getResumeUrls, parallel } from "windmill-client"` ```typescript +/** + * Re-dispatch policy for a failed task. + * + * Every attempt is a step of its own (`fetch`, `fetch#2`, `fetch#3`), and the + * wait between two of them is a durable sleep, so a retrying task holds no + * worker while it backs off. + * + * A workflow sleeps once per round, so tasks backing off in the same fan-out + * wait one after another rather than together: the delay before a fan-out + * retries is the sum of every backoff pending in it, not the longest one, and + * it grows with both the width of the fan-out and `attempts`. Retries with no + * `delay` all go out in a single round. + */ +export interface TaskRetry { + /** Attempts after the first failure: `2` runs the task at most 3 times. + * A whole number from 0 to 100; anything else is rejected where the policy + * is written. */ + attempts: number; + /** Seconds to wait before the first retry. Default 0, retry immediately. + * Sub-second delays are dropped — a durable sleep resolves to the second. */ + delay?: number; + /** Applied to the delay after each attempt: 1 (the default) keeps it + * constant, 2 doubles it. */ + multiplier?: number; + /** Ceiling for the delay in seconds, for a `multiplier` above 1. */ + max_delay?: number; +} + export interface TaskOptions { timeout?: number; tag?: string; @@ -11,6 +39,7 @@ export interface TaskOptions { concurrency_limit?: number; concurrency_key?: string; concurrency_time_window_s?: number; + retry?: TaskRetry; } /** @@ -28,9 +57,11 @@ export async function getResumeUrls(approver?: string, flowLevel?: boolean): Pro * @example * const extract_data = task(async (url: string) => { ... }); * const run_external = task("f/external_script", async (x: number) => { ... }); + * const call_api = task(fetchOrders, { retry: { attempts: 3, delay: 30, multiplier: 2 } }); * * Inside a `workflow()`, calling a task dispatches it as a step. - * Outside a workflow, the function body executes directly. + * Outside a workflow, the function body executes directly and + * {@link TaskOptions} — retry included — does not apply. * * A task runs as its own job, so its result is always encoded as JSON and * decoded back before the caller sees it: a `Date` comes back as a string, a diff --git a/system_prompts/auto-generated/skills/write-script-python3/SKILL.md b/system_prompts/auto-generated/skills/write-script-python3/SKILL.md index 274ffb343e..b91c52d986 100644 --- a/system_prompts/auto-generated/skills/write-script-python3/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-python3/SKILL.md @@ -841,6 +841,22 @@ def parse_sql_client_name(name: str) -> tuple[str, Optional[str]] # decoded back before the caller sees it: a ``datetime`` comes back as a # string, a tuple as a list. # +# ``retry`` re-dispatches the task after a failure, inside ``@workflow`` only. +# Every attempt is a step of its own (``call_api``, ``call_api#2``, ...) and +# the wait between two of them is a durable sleep, so a retrying task holds no +# worker while it backs off. Keys: ``attempts`` (retries after the first +# failure, a whole number from 0 to 100), ``delay`` (seconds before the first +# retry, sub-second delays dropped), ``multiplier`` (applied to the delay +# after each attempt, 1 keeps it constant), ``max_delay`` (ceiling in +# seconds). ``attempts`` is required, and an out-of-range or unknown key is +# rejected where the policy is written. +# +# A workflow sleeps once per round, so tasks backing off in the same fan-out +# wait one after another rather than together: the delay before a fan-out +# retries is the sum of every backoff pending in it, not the longest one, and +# it grows with both the width of the fan-out and ``attempts``. Retries with +# no ``delay`` all go out in a single round. +# # Usage:: # # @task @@ -848,10 +864,15 @@ def parse_sql_client_name(name: str) -> tuple[str, Optional[str]] # # @task(path="f/external_script", timeout=600, tag="gpu") # async def run_external(x: int): ... -def task(_func = None, path: Optional[str] = None, tag: Optional[str] = None, timeout: Optional[int] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None) +# +# @task(retry={"attempts": 3, "delay": 30, "multiplier": 2}) +# async def call_api(payload: dict): ... +def task(_func = None, path: Optional[str] = None, tag: Optional[str] = None, timeout: Optional[int] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None) # Create a task that dispatches to a separate Windmill script. # +# ``retry`` takes the same policy as :func:`task`. +# # Usage:: # # extract = task_script("f/data/extract", timeout=600) @@ -859,10 +880,12 @@ def task(_func = None, path: Optional[str] = None, tag: Optional[str] = None, ti # @workflow # async def main(): # data = await extract(url="https://...") -def task_script(path: str, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None) +def task_script(path: str, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None) # Create a task that dispatches to a separate Windmill flow. # +# ``retry`` takes the same policy as :func:`task`. +# # Usage:: # # pipeline = task_flow("f/etl/pipeline", priority=10) @@ -870,7 +893,7 @@ def task_script(path: str, timeout: Optional[int] = None, tag: Optional[str] = N # @workflow # async def main(): # result = await pipeline(input=data) -def task_flow(path: str, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None) +def task_flow(path: str, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None) # Decorator marking an async function as a workflow-as-code entry point. # diff --git a/system_prompts/auto-generated/skills/write-workflow-as-code/SKILL.md b/system_prompts/auto-generated/skills/write-workflow-as-code/SKILL.md index d02b0a90cf..9da6969e52 100644 --- a/system_prompts/auto-generated/skills/write-workflow-as-code/SKILL.md +++ b/system_prompts/auto-generated/skills/write-workflow-as-code/SKILL.md @@ -246,6 +246,34 @@ A caught failure reads the same whether it came from a task or from a `step()`, Import: `import { workflow, task, taskScript, taskFlow, step, sleep, waitForApproval, getApprovalUrls, getResumeUrls, parallel } from "windmill-client"` ```typescript +/** + * Re-dispatch policy for a failed task. + * + * Every attempt is a step of its own (`fetch`, `fetch#2`, `fetch#3`), and the + * wait between two of them is a durable sleep, so a retrying task holds no + * worker while it backs off. + * + * A workflow sleeps once per round, so tasks backing off in the same fan-out + * wait one after another rather than together: the delay before a fan-out + * retries is the sum of every backoff pending in it, not the longest one, and + * it grows with both the width of the fan-out and `attempts`. Retries with no + * `delay` all go out in a single round. + */ +export interface TaskRetry { + /** Attempts after the first failure: `2` runs the task at most 3 times. + * A whole number from 0 to 100; anything else is rejected where the policy + * is written. */ + attempts: number; + /** Seconds to wait before the first retry. Default 0, retry immediately. + * Sub-second delays are dropped — a durable sleep resolves to the second. */ + delay?: number; + /** Applied to the delay after each attempt: 1 (the default) keeps it + * constant, 2 doubles it. */ + multiplier?: number; + /** Ceiling for the delay in seconds, for a `multiplier` above 1. */ + max_delay?: number; +} + export interface TaskOptions { timeout?: number; tag?: string; @@ -254,6 +282,7 @@ export interface TaskOptions { concurrency_limit?: number; concurrency_key?: string; concurrency_time_window_s?: number; + retry?: TaskRetry; } /** @@ -271,9 +300,11 @@ export async function getResumeUrls(approver?: string, flowLevel?: boolean): Pro * @example * const extract_data = task(async (url: string) => { ... }); * const run_external = task("f/external_script", async (x: number) => { ... }); + * const call_api = task(fetchOrders, { retry: { attempts: 3, delay: 30, multiplier: 2 } }); * * Inside a `workflow()`, calling a task dispatches it as a step. - * Outside a workflow, the function body executes directly. + * Outside a workflow, the function body executes directly and + * {@link TaskOptions} — retry included — does not apply. * * A task runs as its own job, so its result is always encoded as JSON and * decoded back before the caller sees it: a `Date` comes back as a string, a @@ -417,6 +448,22 @@ def get_resume_urls(approver: str = None, flow_level: bool = None) -> dict # decoded back before the caller sees it: a ``datetime`` comes back as a # string, a tuple as a list. # +# ``retry`` re-dispatches the task after a failure, inside ``@workflow`` only. +# Every attempt is a step of its own (``call_api``, ``call_api#2``, ...) and +# the wait between two of them is a durable sleep, so a retrying task holds no +# worker while it backs off. Keys: ``attempts`` (retries after the first +# failure, a whole number from 0 to 100), ``delay`` (seconds before the first +# retry, sub-second delays dropped), ``multiplier`` (applied to the delay +# after each attempt, 1 keeps it constant), ``max_delay`` (ceiling in +# seconds). ``attempts`` is required, and an out-of-range or unknown key is +# rejected where the policy is written. +# +# A workflow sleeps once per round, so tasks backing off in the same fan-out +# wait one after another rather than together: the delay before a fan-out +# retries is the sum of every backoff pending in it, not the longest one, and +# it grows with both the width of the fan-out and ``attempts``. Retries with +# no ``delay`` all go out in a single round. +# # Usage:: # # @task @@ -424,10 +471,15 @@ def get_resume_urls(approver: str = None, flow_level: bool = None) -> dict # # @task(path="f/external_script", timeout=600, tag="gpu") # async def run_external(x: int): ... -def task(_func = None, *, path: Optional[str] = None, tag: Optional[str] = None, timeout: Optional[int] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None) +# +# @task(retry={"attempts": 3, "delay": 30, "multiplier": 2}) +# async def call_api(payload: dict): ... +def task(_func = None, *, path: Optional[str] = None, tag: Optional[str] = None, timeout: Optional[int] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None) # Create a task that dispatches to a separate Windmill script. # +# ``retry`` takes the same policy as :func:`task`. +# # Usage:: # # extract = task_script("f/data/extract", timeout=600) @@ -435,10 +487,12 @@ def task(_func = None, *, path: Optional[str] = None, tag: Optional[str] = None, # @workflow # async def main(): # data = await extract(url="https://...") -def task_script(path: str, *, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None) +def task_script(path: str, *, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None) # Create a task that dispatches to a separate Windmill flow. # +# ``retry`` takes the same policy as :func:`task`. +# # Usage:: # # pipeline = task_flow("f/etl/pipeline", priority=10) @@ -446,7 +500,7 @@ def task_script(path: str, *, timeout: Optional[int] = None, tag: Optional[str] # @workflow # async def main(): # result = await pipeline(input=data) -def task_flow(path: str, *, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None) +def task_flow(path: str, *, timeout: Optional[int] = None, tag: Optional[str] = None, cache_ttl: Optional[int] = None, priority: Optional[int] = None, concurrency_limit: Optional[int] = None, concurrency_key: Optional[str] = None, concurrency_time_window_s: Optional[int] = None, retry: Optional[dict] = None) # Decorator marking an async function as a workflow-as-code entry point. # diff --git a/system_prompts/generate.py b/system_prompts/generate.py index 6d99be1036..439fb7df9f 100644 --- a/system_prompts/generate.py +++ b/system_prompts/generate.py @@ -1458,9 +1458,10 @@ def extract_wac_ts_sdk(ts_content: str) -> str: return '' declarations = [] - task_options = _extract_ts_interface(ts_content, 'TaskOptions') - if task_options: - declarations.append(task_options) + for interface_name in ('TaskRetry', 'TaskOptions'): + interface = _extract_ts_interface(ts_content, interface_name) + if interface: + declarations.append(interface) for function_name in WAC_TS_FUNCTIONS: signature = _extract_ts_exported_function(ts_content, function_name) diff --git a/typescript-client/build.sh b/typescript-client/build.sh index 874e61e070..40c19d873c 100755 --- a/typescript-client/build.sh +++ b/typescript-client/build.sh @@ -46,7 +46,7 @@ cp "${script_dirpath}/sqlUtils.ts" "${script_dirpath}/src/" echo "" >> "${script_dirpath}/src/index.ts" echo 'export type { DenoS3LightClientSettings } from "./s3Types";' >> "${script_dirpath}/src/index.ts" echo "" >> "${script_dirpath}/src/index.ts" -echo 'export { type Base64, setClient, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, setProgress, getProgress, getState, getIdToken, denoS3LightClientSettings, cancelJob, loadS3FileStream, loadS3File, writeS3File, deleteS3File, signS3Objects, signS3Object, getPresignedS3PublicUrls, getPresignedS3PublicUrl, task, taskScript, taskFlow, workflow, step, sleep, parallel, waitForApproval, getApprovalUrls, type TaskOptions, type Jsonified, type JsonifiedFn, WorkflowCtx, _workflowCtx, setWorkflowCtx, StepSuspend, runScript, runScriptAsync, runScriptByPath, runScriptByHash, runScriptByPathAsync, runScriptByHashAsync, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, type Sql, requestInteractiveTeamsApproval, appendToResultStream, streamResult, datatable, ducklake, upsertPartition, appendPartition, type DucklakeMaterializeOptions, type SqlStatement, type DatatableSqlTemplateFunction, type SqlTemplateFunction, type S3Object, type S3ObjectRecord, type S3ObjectURI, commitKafkaOffsets } from "./client";' >> "${script_dirpath}/src/index.ts" +echo 'export { type Base64, setClient, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, setProgress, getProgress, getState, getIdToken, denoS3LightClientSettings, cancelJob, loadS3FileStream, loadS3File, writeS3File, deleteS3File, signS3Objects, signS3Object, getPresignedS3PublicUrls, getPresignedS3PublicUrl, task, taskScript, taskFlow, workflow, step, sleep, parallel, waitForApproval, getApprovalUrls, type TaskOptions, type TaskRetry, type Jsonified, type JsonifiedFn, WorkflowCtx, _workflowCtx, setWorkflowCtx, StepSuspend, runScript, runScriptAsync, runScriptByPath, runScriptByHash, runScriptByPathAsync, runScriptByHashAsync, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, type Sql, requestInteractiveTeamsApproval, appendToResultStream, streamResult, datatable, ducklake, upsertPartition, appendPartition, type DucklakeMaterializeOptions, type SqlStatement, type DatatableSqlTemplateFunction, type SqlTemplateFunction, type S3Object, type S3ObjectRecord, type S3ObjectURI, commitKafkaOffsets } from "./client";' >> "${script_dirpath}/src/index.ts" # Build default export by combining client utilities + services # This preserves backward compatibility for `import wmill from "windmill-client"` diff --git a/typescript-client/client.ts b/typescript-client/client.ts index 105641eee4..480123f3e4 100644 --- a/typescript-client/client.ts +++ b/typescript-client/client.ts @@ -1682,6 +1682,33 @@ export type JsonifiedFn Promise> = ( ...args: Parameters ) => Promise>>>; +/** Re-dispatch policy for a failed task. + * + * Every attempt is a step of its own (`fetch`, `fetch#2`, `fetch#3`), and the + * wait between two of them is a durable sleep, so a retrying task holds no + * worker while it backs off. + * + * A workflow sleeps once per round, so tasks backing off in the same fan-out + * wait one after another rather than together: the delay before a fan-out + * retries is the sum of every backoff pending in it, not the longest one, and + * it grows with both the width of the fan-out and `attempts`. Retries with no + * `delay` all go out in a single round. + */ +export interface TaskRetry { + /** Attempts after the first failure: `2` runs the task at most 3 times. + * A whole number from 0 to 100; anything else is rejected where the policy + * is written. */ + attempts: number; + /** Seconds to wait before the first retry. Default 0, retry immediately. + * Sub-second delays are dropped — a durable sleep resolves to the second. */ + delay?: number; + /** Applied to the delay after each attempt: 1 (the default) keeps it + * constant, 2 doubles it. */ + multiplier?: number; + /** Ceiling for the delay in seconds, for a `multiplier` above 1. */ + max_delay?: number; +} + export interface TaskOptions { timeout?: number; tag?: string; @@ -1690,6 +1717,48 @@ export interface TaskOptions { concurrency_limit?: number; concurrency_key?: string; concurrency_time_window_s?: number; + retry?: TaskRetry; +} + +/** The worker deserializes a sleep into a `u32` of seconds and fails the whole + * job on anything wider, so a delay a multiplier has run away with has to be + * capped here rather than sent. */ +const MAX_SLEEP_SECONDS = 0xffffffff; + +/** Every attempt claims its keys before the first one is dispatched, so an + * unbounded `attempts` is a workflow that hangs allocating rather than a very + * patient one. */ +const MAX_RETRY_ATTEMPTS = 100; + +/** Rejected where the policy is written, so a workflow fails at its first line + * rather than mid-run on a replay. */ +function assertUsableRetry(retry: TaskRetry | undefined): void { + if (retry === undefined) return; + const { attempts } = retry; + if (!Number.isInteger(attempts) || attempts < 0 || attempts > MAX_RETRY_ATTEMPTS) { + throw new Error( + `retry.attempts must be a whole number between 0 and ${MAX_RETRY_ATTEMPTS}, got ${attempts}`, + ); + } +} + +/** How many retries the policy asks for, defended against a value that reached + * `_nextStep` without going through `assertUsableRetry`. */ +function retryAttempts(retry: TaskRetry | undefined): number { + const attempts = Math.trunc(retry?.attempts ?? 0) || 0; + return Math.min(Math.max(attempts, 0), MAX_RETRY_ATTEMPTS); +} + +/** Seconds to wait before retry number `attempt` (0 is the first retry). */ +function retryDelaySeconds(retry: TaskRetry, attempt: number): number { + const base = retry.delay ?? 0; + if (!(base > 0)) return 0; + const grown = base * Math.pow(retry.multiplier ?? 1, attempt); + // Clamping against the ceiling also absorbs the `Infinity` an aggressive + // multiplier reaches within a few attempts. Floor, rather than round, so + // sub-second delays drop the same way they do in the python client. + const seconds = Math.floor(Math.min(retry.max_delay ?? grown, grown, MAX_SLEEP_SECONDS)); + return seconds > 0 ? seconds : 0; } /** A step key travels as one path segment when its URLs are minted, so it must be @@ -1777,59 +1846,103 @@ export class WorkflowCtx { options?: TaskOptions, ): PromiseLike { this._rethrowSwallowed(); - const key = this._allocKey(name || script || "step"); + const stepName = name || script || "step"; + const maxRetries = retryAttempts(options?.retry); - if (key in this.completed) { - const value = this.completed[key]; - if (value && typeof value === "object" && (value as any).__wmill_error) { - const err = taskErrorFromMarker(value, `Task '${name}' failed`); - return { then: (_resolve: any, reject?: any) => { if (reject) reject(err); else throw err; } } as PromiseLike; + // Claimed up front, all of them, and named off the first attempt's key: one + // allocated later would shift the keys of the steps beside it, and a + // `step()` named `t#2` — names are arbitrary — could alias one. Whichever is + // allocated second is the one renamed, identically in every round. + const baseKey = this._allocKey(stepName); + const attemptKeys = [baseKey]; + const backoffKeys: string[] = []; + for (let i = 0; i < maxRetries; i++) { + backoffKeys.push(this._allocKey(`${baseKey}#retry${i + 2}`)); + attemptKeys.push(this._allocKey(`${baseKey}#${i + 2}`)); + } + + // One pass per attempt. Every attempt the checkpoint already holds is + // decided here — a failed one either retries (moving to the next key) or is + // handed back to the body — so the loop always ends at the first attempt + // that has yet to run. + for (let attempt = 0; ; attempt++) { + const key = attemptKeys[attempt]; + + if (key in this.completed) { + const value = this.completed[key]; + if (value && typeof value === "object" && (value as any).__wmill_error) { + if (attempt < maxRetries) { + this._retryBackoff(backoffKeys[attempt], baseKey, options!.retry!, attempt); + continue; + } + const err = taskErrorFromMarker(value, `Task '${name}' failed`); + return { then: (_resolve: any, reject?: any) => { if (reject) reject(err); else throw err; } } as PromiseLike; + } + return { then: (resolve: any) => resolve(value) }; } - return { then: (resolve: any) => resolve(value) }; - } - // If this is a child job executing a specific step, return null to signal - // that the task wrapper should run the inner function directly - if (this._executingKey === key) { - return { then: (resolve: any) => resolve(null), _execute_directly: true } as any; - } + // If this is a child job executing a specific step, return null to signal + // that the task wrapper should run the inner function directly + if (this._executingKey === key) { + return { then: (resolve: any) => resolve(null), _execute_directly: true } as any; + } - // In child job mode (_executingKey is set), non-matching uncompleted steps - // should never resolve or throw — the matching step will throw step_complete - // which terminates the workflow. Returning a never-resolving thenable prevents - // race conditions where a non-matching step's StepSuspend fires before step_complete. - if (this._executingKey !== null) { - return { then: () => new Promise(() => {}) }; - } + // In child job mode (_executingKey is set), non-matching uncompleted steps + // should never resolve or throw — the matching step will throw step_complete + // which terminates the workflow. Returning a never-resolving thenable prevents + // race conditions where a non-matching step's StepSuspend fires before step_complete. + if (this._executingKey !== null) { + return { then: () => new Promise(() => {}) }; + } - const stepInfo: any = { name: name || key, script: script || key, args, key, dispatch_type }; - if (options) { - if (options.timeout !== undefined) stepInfo.timeout = options.timeout; - if (options.tag !== undefined) stepInfo.tag = options.tag; - if (options.cache_ttl !== undefined) stepInfo.cache_ttl = options.cache_ttl; - if (options.priority !== undefined) stepInfo.priority = options.priority; - if (options.concurrency_limit !== undefined) stepInfo.concurrent_limit = options.concurrency_limit; - if (options.concurrency_key !== undefined) stepInfo.concurrency_key = options.concurrency_key; - if (options.concurrency_time_window_s !== undefined) stepInfo.concurrency_time_window_s = options.concurrency_time_window_s; + const stepInfo: any = { name: name || key, script: script || key, args, key, dispatch_type }; + if (options) { + if (options.timeout !== undefined) stepInfo.timeout = options.timeout; + if (options.tag !== undefined) stepInfo.tag = options.tag; + if (options.cache_ttl !== undefined) stepInfo.cache_ttl = options.cache_ttl; + if (options.priority !== undefined) stepInfo.priority = options.priority; + if (options.concurrency_limit !== undefined) stepInfo.concurrent_limit = options.concurrency_limit; + if (options.concurrency_key !== undefined) stepInfo.concurrency_key = options.concurrency_key; + if (options.concurrency_time_window_s !== undefined) stepInfo.concurrency_time_window_s = options.concurrency_time_window_s; + } + this.pending.push(stepInfo); + return { + then: (): never => { + // Only the first .then() call throws with all accumulated steps. + // Subsequent calls (e.g. from Promise.all resolving other thenables) + // also throw (they'll be caught by the same handler). + if (this._suspended) return new Promise(() => {}) as never; + this._suspended = true; + const steps = [...this.pending]; + this.pending = []; + const names = steps.map(s => s.name).join(", "); + console.log(`\n--- WAC: ${names} ---`); + this._raiseSuspend({ + mode: steps.length > 1 ? "parallel" : "sequential", + steps, + }); + }, + }; } - this.pending.push(stepInfo); - return { - then: (): never => { - // Only the first .then() call throws with all accumulated steps. - // Subsequent calls (e.g. from Promise.all resolving other thenables) - // also throw (they'll be caught by the same handler). - if (this._suspended) return new Promise(() => {}) as never; - this._suspended = true; - const steps = [...this.pending]; - this.pending = []; - const names = steps.map(s => s.name).join(", "); - console.log(`\n--- WAC: ${names} ---`); - this._raiseSuspend({ - mode: steps.length > 1 ? "parallel" : "sequential", - steps, - }); - }, - }; + } + + /** Wait out the backoff between two attempts of a retried task, as a durable + * sleep, and return once there is nothing to wait for — no delay configured, + * or the sleep already in the checkpoint. + * + * Raises where it stands, the way `_sleep` does, rather than handing back a + * thenable: a task call the body never awaits is still dispatched (the runner + * flushes `pending`), so a backoff that only fired when awaited would drop + * the retry and let the round report the workflow complete. */ + private _retryBackoff(key: string, baseKey: string, retry: TaskRetry, attempt: number): void { + const seconds = retryDelaySeconds(retry, attempt); + if (seconds < 1) return; + if (key in this.completed) return; + // Child mode never raises: the parent dispatched this child only after its + // own round had slept, so the loop moves on to the attempt being executed. + if (this._executingKey !== null) return; + console.log(`\n--- WAC: sleep(${key}, ${seconds}s) before retrying ${baseKey} ---`); + this._raiseSuspend({ mode: "sleep", key, seconds, steps: [] }); } /** Return and clear any pending (unawaited) steps. */ _flushPending(): Array<{ name: string; script: string; args: Record; key: string; dispatch_type: string }> { @@ -2145,9 +2258,11 @@ export async function step( * @example * const extract_data = task(async (url: string) => { ... }); * const run_external = task("f/external_script", async (x: number) => { ... }); + * const call_api = task(fetchOrders, { retry: { attempts: 3, delay: 30, multiplier: 2 } }); * * Inside a `workflow()`, calling a task dispatches it as a step. - * Outside a workflow, the function body executes directly. + * Outside a workflow, the function body executes directly and + * {@link TaskOptions} — retry included — does not apply. * * A task runs as its own job, so its result is always encoded as JSON and * decoded back before the caller sees it: a `Date` comes back as a string, a @@ -2171,6 +2286,8 @@ export function task Promise>( taskOptions = maybeFnOrOptions as TaskOptions | undefined; } + assertUsableRetry(taskOptions?.retry); + const taskName = fn.name || taskPath || ""; // NOT async — in workflow context we return the thenable directly so that @@ -2255,6 +2372,7 @@ export function task Promise>( * // inside workflow: await extract({ url: "https://..." }) */ export function taskScript(path: string, options?: TaskOptions): (...args: any[]) => PromiseLike { + assertUsableRetry(options?.retry); const name = path.split("/").pop() || path; const wrapper = function (...args: any[]) { const ctx: WorkflowCtx | null = _workflowCtx ?? Reflect.get(globalThis, "__wmill_wf_ctx"); @@ -2280,6 +2398,7 @@ export function taskScript(path: string, options?: TaskOptions): (...args: any[] * // inside workflow: await pipeline({ input: data }) */ export function taskFlow(path: string, options?: TaskOptions): (...args: any[]) => PromiseLike { + assertUsableRetry(options?.retry); const name = path.split("/").pop() || path; const wrapper = function (...args: any[]) { const ctx: WorkflowCtx | null = _workflowCtx ?? Reflect.get(globalThis, "__wmill_wf_ctx"); diff --git a/typescript-client/tests/workflow_retry.test.ts b/typescript-client/tests/workflow_retry.test.ts new file mode 100644 index 0000000000..8c84ffb089 --- /dev/null +++ b/typescript-client/tests/workflow_retry.test.ts @@ -0,0 +1,225 @@ +/** + * `TaskOptions.retry`, against the real client. + * + * Run with: bun test typescript-client/tests/workflow_retry.test.ts + * + * Nothing carries a retry across rounds: every round re-derives which attempt + * comes next from the checkpoint alone, so these drive the rounds a worker + * would and assert what it would act on. Imports client.ts itself (a mirror + * would pin the mirror), with the two generated modules stubbed so the import + * works without ./build.sh. + */ +import { expect, test, describe, mock } from "bun:test"; + +mock.module("../services.gen", () => ({ + ResourceService: {}, + VariableService: {}, + JobService: {}, + HelpersService: {}, + AppService: {}, + MetricsService: {}, + OidcService: {}, + UserService: {}, + KafkaTriggerService: {}, +})); +mock.module("../core/OpenAPI", () => ({ + OpenAPI: { BASE: "http://localhost:8000/api", TOKEN: "tok" }, +})); + +const { WorkflowCtx, task, step, setWorkflowCtx, StepSuspend } = await import("../client.ts"); +import { isSuspendSignal } from "../wacError"; + +/** What the failed child job leaves in `completed_steps`. */ +const failed = { + __wmill_error: true, + message: "boom", + step_key: "callApi", + result: { error: { name: "Error", message: "boom" } }, +}; + +/** One round against `completed`, reduced to what the worker acts on. */ +async function round(completed: Record, body: () => Promise): Promise { + const ctx = new WorkflowCtx({ completed_steps: completed } as any); + setWorkflowCtx(ctx); + try { + return { type: "complete", result: await body() }; + } catch (e: any) { + if (isSuspendSignal(e, StepSuspend)) return { type: "suspend", ...e.dispatchInfo }; + return { type: "error", error: e }; + } finally { + setWorkflowCtx(null); + } +} + +describe("task retry", () => { + test("each failure buys a backoff sleep and one more attempt, until attempts run out", async () => { + const callApi = task(async function callApi(x: number) { + return x; + }, { retry: { attempts: 2, delay: 30, multiplier: 2 } }); + const body = () => callApi(1) as Promise; + + let r = await round({}, body); + expect(r.steps.map((s: any) => s.key)).toEqual(["callApi"]); + + const completed: Record = { callApi: failed }; + r = await round(completed, body); + expect(r).toMatchObject({ mode: "sleep", key: "callApi#retry2", seconds: 30 }); + + completed["callApi#retry2"] = null; + r = await round(completed, body); + expect(r.steps.map((s: any) => s.key)).toEqual(["callApi#2"]); + + // the delay grows by the multiplier for the second retry + completed["callApi#2"] = failed; + r = await round(completed, body); + expect(r).toMatchObject({ mode: "sleep", key: "callApi#retry3", seconds: 60 }); + + completed["callApi#retry3"] = null; + r = await round(completed, body); + expect(r.steps.map((s: any) => s.key)).toEqual(["callApi#3"]); + + // attempts spent: the failure reaches the body + completed["callApi#3"] = failed; + r = await round(completed, body); + expect(r.type).toBe("error"); + expect(r.error.name).toBe("TaskError"); + }); + + test("retrying one call of a task does not move the keys of the calls beside it", async () => { + const t = task(async function t(x: number) { + return x; + }, { retry: { attempts: 1 } }); + const body = () => Promise.all([t(1), t(2)]); + + let r = await round({}, body); + expect(r.steps.map((s: any) => s.key)).toEqual(["t", "t_2"]); + + // the first call retries as `t#2`; the second keeps the `t_2` it was + // dispatched under, rather than being read as the first call's retry + r = await round({ t: failed, t_2: 20 }, body); + expect(r.steps.map((s: any) => s.key)).toEqual(["t#2"]); + }); + + test("a task the body never awaits still sleeps and retries", async () => { + // The runner dispatches unawaited task calls by flushing `pending`, so a + // backoff that only fired when awaited would drop the retry silently and + // report the workflow complete. + const fire = task(async function fire() { + return 1; + }, { retry: { attempts: 1, delay: 30 } }); + const body = async () => { + fire(); + return "done"; + }; + + const r = await round({ fire: failed }, body); + expect(r).toMatchObject({ mode: "sleep", key: "fire#retry2", seconds: 30 }); + }); + + // `step()` names are arbitrary strings, so a step really can be called `t#2`. + // Whichever of the two allocates second is the one renamed, and it has to be + // the same one in every round — hence claiming the attempt keys up front. + test("an inline step named like an attempt key, before the task, keeps its key", async () => { + const t = task(async function t(x: number) { + return x; + }, { retry: { attempts: 1 } }); + const body = async () => { + const decoy = await step("t#2", () => "not an attempt"); + return [decoy, await t(1)]; + }; + + const r = await round({ "t#2": "not an attempt", t: failed }, body); + expect(r.steps.map((s: any) => s.key)).toEqual(["t#2_2"]); + }); + + test("an inline step named like an attempt key, after the task, does not stand in for it", async () => { + const t = task(async function t(x: number) { + return x; + }, { retry: { attempts: 1 } }); + const body = async () => { + const pending = t(1); + const decoy = await step("t#2", () => "not an attempt"); + return [decoy, await pending]; + }; + + // Round 1 records the step under the key left over after the task claimed + // `t#2`, so the retry re-dispatches instead of reading the step's value. + let r = await round({}, body); + expect(r).toMatchObject({ mode: "inline_checkpoint", key: "t#2_2" }); + + r = await round({ t: failed, "t#2_2": "not an attempt" }, body); + expect(r.steps.map((s: any) => s.key)).toEqual(["t#2"]); + }); + + test("the child dispatched for an attempt walks the loop past the failure and backoff", async () => { + // The non-matching branch returns a thenable that never resolves, so a + // child that walks the loop wrong parks the run until its timeout. + const t = task(async function t(x: number) { + return x * 10; + }, { retry: { attempts: 1, delay: 30 } }); + const ctx = new WorkflowCtx({ + completed_steps: { t: failed, "t#retry2": null }, + _executing_key: "t#2", + } as any); + setWorkflowCtx(ctx); + try { + const suspend: any = await t(4).then( + () => null, + (e: any) => e, + ); + expect(suspend.dispatchInfo).toMatchObject({ mode: "step_complete", result: 40 }); + } finally { + setWorkflowCtx(null); + } + }); + + test("an out-of-range attempt count is rejected where it is written", () => { + // Each attempt claims its keys before the first one is dispatched, so an + // unbounded count would hang the workflow allocating them. + for (const attempts of [10_000, -1, 2.5, Infinity, NaN]) { + expect(() => + task(async function t(x: number) { + return x; + }, { retry: { attempts } }), + ).toThrow("whole number"); + } + }); + + test("a zero multiplier is honoured, not read as the default of 1", async () => { + const t = task(async function t(x: number) { + return x; + }, { retry: { attempts: 2, delay: 30, multiplier: 0 } }); + const body = () => t(1) as Promise; + + let r = await round({ t: failed }, body); + expect(r).toMatchObject({ mode: "sleep", key: "t#retry2", seconds: 30 }); + + // 30 * 0 — the second retry goes out with no wait at all + r = await round({ t: failed, "t#retry2": null, "t#2": failed }, body); + expect(r.steps.map((s: any) => s.key)).toEqual(["t#3"]); + }); + + test("max_delay caps the backoff a multiplier grows", async () => { + const t = task(async function t(x: number) { + return x; + }, { retry: { attempts: 2, delay: 60, multiplier: 100, max_delay: 300 } }); + const body = () => t(1) as Promise; + + const r = await round({ t: failed, "t#retry2": null, "t#2": failed }, body); + expect(r).toMatchObject({ mode: "sleep", key: "t#retry3", seconds: 300 }); + }); + + test("a retry that succeeds resolves to its value, and later steps keep their own keys", async () => { + // No delay, so the retry is dispatched without a sleep round in between. + const flaky = task(async function flaky(x: number) { + return x; + }, { retry: { attempts: 1 } }); + const double = task(async function double(v: number) { + return v * 2; + }); + const body = async () => double((await flaky(1)) as number); + + const r = await round({ flaky: failed, "flaky#2": 7 }, body); + expect(r.steps).toMatchObject([{ key: "double", args: { v: 7 } }]); + }); +}); From 6860521b4e8f01f589b87a401064f45f56a2e819 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 8 Sep 2026 09:41:05 +0000 Subject: [PATCH 021/144] refresh a dbt column trace with its graph, and stop calling whole ones cut (#11015) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: refresh a dbt column trace with its graph, and stop calling whole ones cut Follow-up to #11014, addressing two findings from the review round that landed after it merged. **A deliberate graph refresh now re-asks for the lineage.** The dedup key held only the workspace, the pin and the seed relations, all of which a redeploy leaves alone — so the pipeline page's Refresh refetched the graph and left the trace as it was, pairing the new version's SQL and columns with the previous one's edges. The key now carries which fetch of the graph is on screen, taken from `graphRes.current`'s identity: it moves on a Refresh, a deploy and a folder switch, and on nothing else, so an editor keystroke still cannot make the pane re-ask. **`truncated` is set only with evidence.** `pending` was read as proof the component had been cut, but it only says a relation's owners have not been asked about yet — and those owners are usually the project already in hand. A project holding more than the expansion budget across unrelated families therefore reported a small, complete component as truncated. The owners query now runs before the budget and round stops, so a trace is called cut only when a project this caller may read is left unread, or when the walk itself was cut. Two smaller things from the same round: a failed lineage request says so instead of rendering the empty trace a project without the analysis pass renders — the two were indistinguishable, and a Refresh now retries it — and `asset_paths` is capped as well as refused when empty. `MAX_HELD_EDGES` is renamed `EXPANSION_EDGE_BUDGET`: it never bounded what its name claimed, since the seeds' own projects are read whole whatever their size. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01NY4kuFy2jAnGEzaCc1CseL * fix: show a failed column-lineage request beside a partial trace, not only instead of one Review-round findings on this PR. The failure line only rendered when the trace had no nodes at all. A ducklake selection whose producers declare column lineage has nodes from the graph the canvas already carries, so a failed dbt request left a trace that rendered and was missing a half — which is the reading the line exists to prevent. It now renders beside a drawn trace as well, and says the trace may be incomplete rather than that nothing loaded. The dbt branch of the details pane also opened on `selectionColumnLoading` but not on the failed state, so a relation with neither SQL nor a column schema fell through to "no inline preview" and the line never rendered at all. Dropped "Refresh to try again": the dbt editor has no Refresh for this, and its recovery is a re-parse or reselecting. The comment on the error handler says both paths again rather than only the one the pipeline page uses. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01NY4kuFy2jAnGEzaCc1CseL * fix: clear a column-lineage failure when the next request goes out Round-2 nits, both reviewers on the same state. `failed` was cleared only when an answer landed, so a retry kept saying the trace may be incomplete while it was being fetched, and a new selection inherited the previous one's failure until its own answer arrived. It is cleared as the request goes out instead. Also documents the bounds on `asset_path` in the two routes that take it: the 1000-relation cap and the at-least-one rule were both enforced and neither was written down, so a caller met them as a 400 with no way to have known. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01NY4kuFy2jAnGEzaCc1CseL --------- Co-authored-by: Claude Opus 5 (1M context) --- ...83bf1585004f9fddee904ab176ac85ba3c094.json | 16 +++++ ...41bdac5c52e50e697c6714b91b9fd3155572c.json | 16 +++++ ...14db3228451391ce11b8a14d7c11d4ee304ba.json | 16 +++++ backend/windmill-api-assets/src/lib.rs | 58 +++++++++------ .../tests/dbt_pinned_graph.rs | 71 +++++++++++++++++++ backend/windmill-api/openapi.yaml | 12 +++- docs/dbt-runtime.md | 22 ++++-- .../AssetGraph/AssetGraphDetailsPane.svelte | 8 ++- .../AssetGraph/ColumnTraceSection.svelte | 39 +++++++--- .../AssetGraph/PipelineGraphEditor.svelte | 5 ++ .../AssetGraph/dbtColumnLineage.svelte.ts | 43 +++++++++-- .../src/lib/components/dbt/DbtEditor.svelte | 1 + .../lib/components/dbt/DbtModelDetails.svelte | 3 + .../(logged)/pipeline/[folder]/+page.svelte | 14 +++- 14 files changed, 281 insertions(+), 43 deletions(-) create mode 100644 backend/.sqlx/query-3162283a5066660e2f31575344083bf1585004f9fddee904ab176ac85ba3c094.json create mode 100644 backend/.sqlx/query-a87dcb8f812b555564d71afcb7141bdac5c52e50e697c6714b91b9fd3155572c.json create mode 100644 backend/.sqlx/query-cec32d42bceaf500ffaf47102cd14db3228451391ce11b8a14d7c11d4ee304ba.json diff --git a/backend/.sqlx/query-3162283a5066660e2f31575344083bf1585004f9fddee904ab176ac85ba3c094.json b/backend/.sqlx/query-3162283a5066660e2f31575344083bf1585004f9fddee904ab176ac85ba3c094.json new file mode 100644 index 0000000000..0937a312d1 --- /dev/null +++ b/backend/.sqlx/query-3162283a5066660e2f31575344083bf1585004f9fddee904ab176ac85ba3c094.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id,\n parent_unique_id, parent_column, child_unique_id,\n child_column, lineage_kind)\n VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000',\n 'model.p.stock', 'sku', 'model.p.stock_daily', 'sku', 'copy')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "3162283a5066660e2f31575344083bf1585004f9fddee904ab176ac85ba3c094" +} diff --git a/backend/.sqlx/query-a87dcb8f812b555564d71afcb7141bdac5c52e50e697c6714b91b9fd3155572c.json b/backend/.sqlx/query-a87dcb8f812b555564d71afcb7141bdac5c52e50e697c6714b91b9fd3155572c.json new file mode 100644 index 0000000000..313dec3743 --- /dev/null +++ b/backend/.sqlx/query-a87dcb8f812b555564d71afcb7141bdac5c52e50e697c6714b91b9fd3155572c.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id,\n resource_type, name, asset_path, tags)\n VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000', 'model.p.stock',\n 'model', 'stock', 'u/a/wh/analytics/stock', '{}'),\n ($1, $2, $3, '00000000-0000-0000-0000-000000000000', 'model.p.stock_daily',\n 'model', 'stock_daily', 'u/a/wh/analytics/stock_daily', '{}')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "a87dcb8f812b555564d71afcb7141bdac5c52e50e697c6714b91b9fd3155572c" +} diff --git a/backend/.sqlx/query-cec32d42bceaf500ffaf47102cd14db3228451391ce11b8a14d7c11d4ee304ba.json b/backend/.sqlx/query-cec32d42bceaf500ffaf47102cd14db3228451391ce11b8a14d7c11d4ee304ba.json new file mode 100644 index 0000000000..2ef26b366b --- /dev/null +++ b/backend/.sqlx/query-cec32d42bceaf500ffaf47102cd14db3228451391ce11b8a14d7c11d4ee304ba.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id,\n parent_unique_id, parent_column, child_unique_id,\n child_column, lineage_kind)\n SELECT $1, $2, $3, '00000000-0000-0000-0000-000000000000',\n 'model.p.raw_orders', 'c' || i, 'model.p.orders', 'c' || i, 'copy'\n FROM generate_series(1, 100000) i", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "cec32d42bceaf500ffaf47102cd14db3228451391ce11b8a14d7c11d4ee304ba" +} diff --git a/backend/windmill-api-assets/src/lib.rs b/backend/windmill-api-assets/src/lib.rs index 4b86013e7d..89e3c7c6b2 100644 --- a/backend/windmill-api-assets/src/lib.rs +++ b/backend/windmill-api-assets/src/lib.rs @@ -1037,6 +1037,11 @@ impl ColumnLineageQuery { "at least one asset_path is required".to_string(), )); } + if asset_paths.len() > MAX_ASKED_RELATIONS { + return Err(windmill_common::error::Error::BadRequest(format!( + "at most {MAX_ASKED_RELATIONS} asset_path values may be asked about at once" + ))); + } Ok(ColumnLineageQuery { asset_paths, dbt_script_hash }) } } @@ -1066,14 +1071,22 @@ const MAX_TRACE_EDGES: usize = 5_000; /// bound says. const MAX_OWNER_ROUNDS: usize = 8; -/// How many edges one trace may HOLD while walking, as opposed to answer with. -/// A project's edges arrive whole — the walk decides what is in the component, -/// so a `LIMIT` on the fetch would cut an arbitrary set that need not even -/// contain the asked-for relation — and a project is bounded at ingest by -/// `MAX_COLUMN_EDGES`, which is 200k. Rounds are what this bounds: reading a -/// second project's worth on top of an already outsized first one buys nothing, -/// since the walk is going to stop at `MAX_TRACE_EDGES` regardless. -const MAX_HELD_EDGES: usize = 100_000; +/// How many edges a trace may already hold before it stops looking for projects +/// it has not read. NOT a bound on what one trace fetches: the seeds' own +/// projects are read whole whatever their size, because reading them IS the +/// answer, and a project's edges arrive whole in any case — the walk is what +/// decides which of them are in the component, so a `LIMIT` would cut a set that +/// need not contain the asked-for relation at all. What a single fetch is +/// bounded by is the ingest's `MAX_COLUMN_EDGES` per version. This bounds the +/// EXPANSION on top of that: reading a further project's worth once a trace is +/// already this size buys nothing the walk will not cut at `MAX_TRACE_EDGES`. +const EXPANSION_EDGE_BUDGET: usize = 100_000; + +/// How many relations one request may ask about. Generous: the pipeline page +/// sends every dbt relation the selection's own producer lineage reaches, which +/// is a handful even in a large folder. It exists so a crafted request cannot +/// hand `= ANY($2)` an arbitrarily long array. +const MAX_ASKED_RELATIONS: usize = 1_000; /// A stored project graph: a deployed version, or one job's snapshot of it. /// `script_hash` is NULL for an editor buffer's parse, which names no version. @@ -1140,8 +1153,9 @@ pub async fn dbt_column_lineage_for( let mut answer: Vec = Vec::new(); let mut pending: Vec = seeds.iter().cloned().collect(); let mut truncated = false; + let mut rounds = 0usize; - for _ in 0..MAX_OWNER_ROUNDS { + loop { if pending.is_empty() { break; } @@ -1232,9 +1246,22 @@ pub async fn dbt_column_lineage_for( .map(|o| (o.script_path, o.script_hash, o.job_id)) .filter(|k| read.insert(k.clone())) .collect(); + // Nothing left this caller may read and has not read: the component is + // whole, however many relations were still waiting to be asked about. + // Their owners are projects already in hand. if fresh.is_empty() { break; } + // From here a project exists that this answer will not contain, so the + // two stops below are cuts and are reported as such. Deciding it after + // the owners query rather than before is what keeps a big project's + // small component from being called truncated: `pending` alone only says + // a relation has not been ASKED about, not that anything was left out. + rounds += 1; + if rounds > MAX_OWNER_ROUNDS || edges.len() >= EXPANSION_EDGE_BUDGET { + truncated = true; + break; + } let fresh_paths: Vec = fresh.iter().map(|k| k.0.clone()).collect(); let fresh_hashes: Vec> = fresh.iter().map(|k| k.1).collect(); let fresh_jobs: Vec = fresh.iter().map(|k| k.2).collect(); @@ -1313,20 +1340,9 @@ pub async fn dbt_column_lineage_for( .collect::>() .into_iter() .collect(); - // Stop discovering projects once the held set is outsized. The tail - // below reports what that leaves unresolved, and reports nothing when - // the walk had already reached everything. - if edges.len() >= MAX_HELD_EDGES { - break; - } } tx.commit().await?; - // Out of rounds with relations still unresolved: more of the component - // exists, which is what hitting the edge bound also means. - Ok(Json(ColumnLineageResponse { - truncated: truncated || !pending.is_empty(), - edges: answer, - })) + Ok(Json(ColumnLineageResponse { truncated, edges: answer })) } struct WalkedComponent { diff --git a/backend/windmill-api-assets/tests/dbt_pinned_graph.rs b/backend/windmill-api-assets/tests/dbt_pinned_graph.rs index 75299f4d9f..ebeee49ecc 100644 --- a/backend/windmill-api-assets/tests/dbt_pinned_graph.rs +++ b/backend/windmill-api-assets/tests/dbt_pinned_graph.rs @@ -940,3 +940,74 @@ async fn a_component_past_the_bound_is_cut_and_says_so(db: Pool) { assert_eq!(body["edges"].as_array().unwrap().len(), 5000); assert_eq!(body["truncated"], serde_json::json!(true)); } + +/// The other side of the same flag: a project big enough to stop the expansion +/// still answers a small component WHOLE, and must not claim it was cut. +/// +/// Reaching the size that stops expansion says nothing on its own — nor does a +/// relation whose owners have not been asked about, since those owners are +/// usually the project already in hand. Only a project this caller may read and +/// this answer does not contain is a cut. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn a_whole_component_in_a_big_project_is_not_called_cut(db: Pool) { + seed(&db, uuid::Uuid::from_u128(7)).await; + seed_deployed_orders(&db).await; + // A second family, two relations and one edge, sharing no column with the + // first — the whole of what a selection on it should return. + sqlx::query!( + "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id, + resource_type, name, asset_path, tags) + VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000', 'model.p.stock', + 'model', 'stock', 'u/a/wh/analytics/stock', '{}'), + ($1, $2, $3, '00000000-0000-0000-0000-000000000000', 'model.p.stock_daily', + 'model', 'stock_daily', 'u/a/wh/analytics/stock_daily', '{}')", + WS, + PATH, + HASH, + ) + .execute(&db) + .await + .unwrap(); + sqlx::query!( + "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id, + parent_unique_id, parent_column, child_unique_id, + child_column, lineage_kind) + VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000', + 'model.p.stock', 'sku', 'model.p.stock_daily', 'sku', 'copy')", + WS, + PATH, + HASH, + ) + .execute(&db) + .await + .unwrap(); + // And enough unrelated families beside them to pass the expansion budget. + sqlx::query!( + "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id, + parent_unique_id, parent_column, child_unique_id, + child_column, lineage_kind) + SELECT $1, $2, $3, '00000000-0000-0000-0000-000000000000', + 'model.p.raw_orders', 'c' || i, 'model.p.orders', 'c' || i, 'copy' + FROM generate_series(1, 100000) i", + WS, + PATH, + HASH, + ) + .execute(&db) + .await + .unwrap(); + + let admin = ApiAuthed { is_admin: true, ..outsider() }; + let body = column_lineage(&db, &admin, &["u/a/wh/analytics/stock_daily"], None).await; + assert_eq!( + body["edges"], + serde_json::json!([{ + "from_asset_path": "u/a/wh/analytics/stock", + "from_column": "sku", + "to_asset_path": "u/a/wh/analytics/stock_daily", + "to_column": "sku", + "kind": "copy", + }]), + ); + assert_eq!(body["truncated"], serde_json::json!(false)); +} diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index e2bb8c6a1a..4abd5f4dff 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -24479,9 +24479,13 @@ paths: required: true description: > The `dbt://` relations whose lineage to return. Repeated, once per - relation, and answered as one union. + relation, and answered as one union. At least one, and at most 1000 — + a request naming none, or more than that, is refused rather than + answered with an empty component. schema: type: array + maxItems: 1000 + minItems: 1 items: type: string - name: dbt_script_hash @@ -24728,9 +24732,13 @@ paths: required: true description: > The `dbt://` relations whose lineage to return. Repeated, once per - relation, and answered as one union. + relation, and answered as one union. At least one, and at most 1000 — + a request naming none, or more than that, is refused rather than + answered with an empty component. schema: type: array + maxItems: 1000 + minItems: 1 items: type: string responses: diff --git a/docs/dbt-runtime.md b/docs/dbt-runtime.md index 926948a676..5fec1e42f7 100644 --- a/docs/dbt-runtime.md +++ b/docs/dbt-runtime.md @@ -1689,11 +1689,23 @@ served — a synthetic 3000-model project whose models share a column returns 58 direct edges and 7.3MB. The walk is breadth-first from the asked-for relations and stops at 5000 edges, so what survives is the part nearest the selection rather than an arbitrary slice, and `truncated` says so: a trace that stops short is -otherwise indistinguishable from one that ends. The FETCH is not bounded the same -way — a project's edges arrive whole, because the walk is what decides which of -them are in the component, and a `LIMIT` would cut a set that need not contain -the asked-for relation at all. What bounds it is the ingest's own cap per version -plus a stop on discovering further projects once the held set is outsized. +otherwise indistinguishable from one that ends. + +`truncated` is set only where a project this caller may read is left unread, or +where the walk itself was cut. Neither the expansion's size budget nor a relation +whose owners were never asked about is evidence on its own: a big project's small +component is answered whole, and the relations still waiting to be asked about at +the end are almost always owned by the project already in hand. The order the +rounds run in is what makes that decidable — the owners query comes before the +budget check, so the stop happens with a named unread project rather than a +suspicion of one. + +The FETCH is not bounded the way the answer is: a project's edges arrive whole, +because the walk is what decides which of them are in the component, and a +`LIMIT` would cut a set that need not contain the asked-for relation at all. The +seeds' own projects are therefore read whatever their size — reading them is the +answer. What bounds a single fetch is the ingest's own cap per version; the +budget bounds only the expansion on top of it. The walk is in Rust rather than a recursive CTE. `EXPLAIN ANALYZE` on that same project measured 1243ms against 59ms for the query alone: a CTE has no index to diff --git a/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte b/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte index 005e3a3ffb..4b252b9338 100644 --- a/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte @@ -163,6 +163,9 @@ /** The lineage reaches past what the graph holds: the API cut it at the * part nearest the selection. */ selectionColumnTruncated?: boolean + /** That trace could not be fetched. Distinguished from an empty one: a + * project without the analysis pass draws nothing either. */ + selectionColumnFailed?: boolean /** dbt provenance of the selected relation, when a dbt project * materializes it — carries the model's own SQL. */ selectionDbt?: DbtAssetProvenance @@ -298,6 +301,7 @@ selectionColumnGraph, selectionColumnLoading = false, selectionColumnTruncated = false, + selectionColumnFailed = false, selectionDbt, schemaCanEvolve = true, selectionForkMaterialization = undefined, @@ -1250,13 +1254,14 @@ targetLabel={selection.path} loading={selectionColumnLoading} truncated={selectionColumnTruncated} + failed={selectionColumnFailed} />
{/key} - {:else if selectionDbt && (selectionDbt.raw_code || selectionDbtHasColumns || selectionColumnNodes.length > 0 || selectionColumnLoading)} + {:else if selectionDbt && (selectionDbt.raw_code || selectionDbtHasColumns || selectionColumnNodes.length > 0 || selectionColumnLoading || selectionColumnFailed)} +{#snippet failure()} +
+ Part of this column lineage could not be loaded, so the trace may be incomplete. +
+{/snippet} + {#if loading && nodes.length === 0}
Loading column lineage
-{:else if graph && nodes.length > 0} +{:else if nodes.length === 0} + {#if failed} +
{@render failure()}
+ {/if} +{:else if graph}
+ {#if failed} + {@render failure()} + {/if} {#if truncated}
Showing the part of the trace nearest this relation. The lineage reaches further than one diff --git a/frontend/src/lib/components/assets/AssetGraph/PipelineGraphEditor.svelte b/frontend/src/lib/components/assets/AssetGraph/PipelineGraphEditor.svelte index 721fafb6be..b71a51ea0a 100644 --- a/frontend/src/lib/components/assets/AssetGraph/PipelineGraphEditor.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/PipelineGraphEditor.svelte @@ -80,6 +80,7 @@ selectionColumnGraph, selectionColumnLoading = false, selectionColumnTruncated = false, + selectionColumnFailed = false, selectionDbt, schemaCanEvolve = true, selectionForkMaterialization = undefined, @@ -191,6 +192,9 @@ selectionColumnLoading?: boolean /** That trace cut at the part nearest the selection. */ selectionColumnTruncated?: boolean + /** That trace could not be fetched. Distinguished from an empty one: a + * project without the analysis pass draws nothing either. */ + selectionColumnFailed?: boolean /** dbt provenance of the selected relation — carries its SQL. */ selectionDbt?: DbtAssetProvenance schemaCanEvolve?: boolean @@ -525,6 +529,7 @@ {selectionColumnGraph} {selectionColumnLoading} {selectionColumnTruncated} + {selectionColumnFailed} {selectionDbt} {schemaCanEvolve} {selectionForkMaterialization} diff --git a/frontend/src/lib/components/assets/AssetGraph/dbtColumnLineage.svelte.ts b/frontend/src/lib/components/assets/AssetGraph/dbtColumnLineage.svelte.ts index 189ba05b5d..477ed630b1 100644 --- a/frontend/src/lib/components/assets/AssetGraph/dbtColumnLineage.svelte.ts +++ b/frontend/src/lib/components/assets/AssetGraph/dbtColumnLineage.svelte.ts @@ -20,6 +20,9 @@ export type DbtColumnLineageState = { /** The component reaches past what `graph` holds — the API cut it at the * part nearest the selection. */ readonly truncated: boolean + /** The request failed, so `graph` is empty for a reason that is not "this + * project has no column lineage". */ + readonly failed: boolean } function fetchLineage( @@ -56,10 +59,18 @@ export function useDbtColumnLineage(args: { assetPaths: () => string[] /** The graph on screen, so the lineage describes the same project. */ pin?: () => DbtGraphPin | undefined + /** Which fetch of that graph is on screen. It changes when the view goes and + * gets the graph again — a Refresh, a deploy — and asking again is the whole + * point: the relation, the pin and the seeds are all unchanged by a + * redeploy, so without this the pane would pair a freshly fetched model's + * SQL and columns with the edges of the version before it. It is also what + * retries a request that failed. */ + generation?: () => unknown }): DbtColumnLineageState { let graph = $state(EMPTY_COLUMN_GRAPH) let loading = $state(false) let truncated = $state(false) + let failed = $state(false) // The question the state in hand answers, and a counter deciding which answer // is still wanted. Neither is a cache of edges: the API returns a whole @@ -71,36 +82,55 @@ export function useDbtColumnLineage(args: { const workspace = args.workspace() const paths = [...new Set(args.assetPaths())].sort() const pin = args.pin?.() - const question = JSON.stringify([workspace, pin?.jobId, pin?.scriptHash, paths]) + const question = JSON.stringify([ + workspace, + pin?.jobId, + pin?.scriptHash, + args.generation?.() ?? null, + paths + ]) // A selection re-derived from a graph that polled is the same question. Not // asking it again is what keeps a run page from refetching a component's - // worth of edges every poll to redraw what is already on screen. + // worth of edges every poll to redraw what is already on screen — while a + // graph the view deliberately went and fetched moves `generation`, so that + // IS a new question. if (question === asked) return asked = question const id = ++latest if (!workspace || paths.length === 0) { graph = EMPTY_COLUMN_GRAPH truncated = false + failed = false loading = false return } + // The previous question's failure is not this one's. Cleared as the request + // goes out rather than when it lands, or a retry keeps saying the trace is + // incomplete while it is being fetched, and a new selection inherits the + // last one's failure until its own answer arrives. + failed = false loading = true fetchLineage(workspace, paths, pin).then( (r) => { if (id !== latest) return graph = buildDbtColumnGraph(r?.edges ?? []) truncated = r?.truncated ?? false + failed = false loading = false }, // Lineage annotates a graph that renders without it, so a failed fetch // leaves that branch unexpanded rather than putting an error over the - // model. Not retried on its own: the effect reruns whenever the canvas - // does, and a failing endpoint would then be asked once per redraw. - // Selecting another node and back asks again. + // model — but it SAYS so, because an empty trace is what a project + // without the analysis pass looks like, and the two must not read + // alike. Not retried on its own: the effect reruns whenever the canvas + // redraws, and a failing endpoint would then be asked once per redraw. + // Selecting another node and back asks again, and where a caller + // passes `generation`, so does going and fetching the graph. () => { if (id !== latest) return graph = EMPTY_COLUMN_GRAPH truncated = false + failed = true loading = false } ) @@ -115,6 +145,9 @@ export function useDbtColumnLineage(args: { }, get truncated() { return truncated + }, + get failed() { + return failed } } } diff --git a/frontend/src/lib/components/dbt/DbtEditor.svelte b/frontend/src/lib/components/dbt/DbtEditor.svelte index 469536f305..4b30f8fa28 100644 --- a/frontend/src/lib/components/dbt/DbtEditor.svelte +++ b/frontend/src/lib/components/dbt/DbtEditor.svelte @@ -586,6 +586,7 @@ columnGraph={selectionColumnGraph} columnLoading={columnLineage.loading} columnTruncated={columnLineage.truncated} + columnFailed={columnLineage.failed} onOpenFile={open} onClose={() => (graphSelection = undefined)} /> diff --git a/frontend/src/lib/components/dbt/DbtModelDetails.svelte b/frontend/src/lib/components/dbt/DbtModelDetails.svelte index d3dd25d04a..5863d00a59 100644 --- a/frontend/src/lib/components/dbt/DbtModelDetails.svelte +++ b/frontend/src/lib/components/dbt/DbtModelDetails.svelte @@ -43,6 +43,7 @@ columnGraph, columnLoading = false, columnTruncated = false, + columnFailed = false, onOpenFile, onClose }: { @@ -57,6 +58,7 @@ columnGraph?: ColumnLineageGraph columnLoading?: boolean columnTruncated?: boolean + columnFailed?: boolean onOpenFile?: (path: string) => void onClose?: () => void } = $props() @@ -273,6 +275,7 @@ targetLabel={dbt.unique_id} loading={columnLoading} truncated={columnTruncated} + failed={columnFailed} /> {#if showRows && preview} diff --git a/frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte b/frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte index 4dc68cd9ab..b4ddb8350c 100644 --- a/frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte @@ -2019,9 +2019,20 @@ } return [...paths] }) + // Which fetch of the folder graph is on screen. `graphRes.current` is a new + // object per fetch, so this moves on a Refresh, a deploy and a folder switch + // — and on nothing else, which is what keeps a keystroke in the editor from + // re-asking. Without it a redeploy would leave the pane pairing the new + // version's SQL and columns with the old version's edges, since the relation, + // the pin and the seeds are all unchanged by one. + let graphGeneration = $state(0) + $effect(() => { + if (graphRes.current) untrack(() => graphGeneration++) + }) const dbtColumnLineage = useDbtColumnLineage({ workspace: () => $workspaceStore, - assetPaths: () => dbtSeedPaths + assetPaths: () => dbtSeedPaths, + generation: () => graphGeneration }) // One graph across both, so a trace crosses the dbt/ducklake boundary in // either direction rather than stopping at it. @@ -2630,6 +2641,7 @@ selectionColumnGraph={pe.activeDraft ? EMPTY_COLUMN_GRAPH : columnGraph} selectionColumnLoading={dbtColumnLineage.loading} selectionColumnTruncated={dbtColumnLineage.truncated} + selectionColumnFailed={dbtColumnLineage.failed} {schemaCanEvolve} {selectionForkMaterialization} {schemaContractContext} From 9444049d6013c77a5f25f01a736eb5cd741fb3e6 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Tue, 8 Sep 2026 11:54:43 +0200 Subject: [PATCH 022/144] feat: bring gitlab repositories to parity for git sync (#10938) * feat: track and rotate gitlab git-sync repository tokens * chore: point ee-repo-ref at the gitlab credential branch * fix: strip server-owned credential status and correct expiry copy * fix: gate credential maintenance on enterprise and alert on stalled renewal * fix: alert on an auto-renewed token only once it has actually expired * feat: receive gitlab push webhooks for instant git sync pull * feat: open gitlab merge requests and post diff previews on them Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01C1xHmkxuxYb1GYvth1BS75 * fix: keep gitlab merge request previews out of the project's own pipeline Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01C1xHmkxuxYb1GYvth1BS75 * fix: bound the credential maintenance pass and gate the gitlab picker on a license Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01C1xHmkxuxYb1GYvth1BS75 * fix: create the gitlab picker's variable in the edited workspace Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01C1xHmkxuxYb1GYvth1BS75 * fix: make the gitlab picker's variable path collision-resistant Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01C1xHmkxuxYb1GYvth1BS75 * docs: state the gitlab scope and rotation facts the code relies on Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01C1xHmkxuxYb1GYvth1BS75 * fix: resolve the check marker's repository from its path, not a stored url Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01C1xHmkxuxYb1GYvth1BS75 * fix: refuse to finish a check whose repository has been repointed Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01C1xHmkxuxYb1GYvth1BS75 * fix: trust a check marker's captured url when it carries no identity Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01C1xHmkxuxYb1GYvth1BS75 * fix: recreate a missing webhook from credential maintenance Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01C1xHmkxuxYb1GYvth1BS75 * docs: state that relative-url gitlab installs are out of scope Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01C1xHmkxuxYb1GYvth1BS75 * fix: keep credential status out of exports and clear stale webhook warnings Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01C1xHmkxuxYb1GYvth1BS75 * fix: refuse an unprovable check and guard the picker on the stored repository Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01C1xHmkxuxYb1GYvth1BS75 * fix: re-check the picker's target path at the moment it is written Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01C1xHmkxuxYb1GYvth1BS75 * fix: snapshot the picker's inputs before it starts writing Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01C1xHmkxuxYb1GYvth1BS75 * docs: recommend a project access token per repository Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01C1xHmkxuxYb1GYvth1BS75 * [ee] feat: keep the git-sync credential in workspace settings * [ee] fix: drop a removed repo's credential and honor the workspace override * [ee] fix: resolve a fork's git credential from its whole ancestry * [ee] refactor: reuse fork_ancestor_chain instead of a second ancestry walk * [ee] fix: resolve an app installation from the whole ancestry, not the parent * [ee] revert: keep the app installation fallback at one level * fix: store the git credential only once the resource is saved * fix: keep a repository's credential when it leaves git sync settings * docs: cut the gitlab picker's token guidance down to what it needs * feat: mark a repository whose credential windmill holds * fix: ignore the managed-credential marker when the url carries a token * docs: drop the picker's setup alert for a line by the token field * feat: replace a repository's stored token from its resource * fix: store a picked credential for its own workspace, before the resource * refactor: key a stored git credential by its repository, not its resource Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01C1xHmkxuxYb1GYvth1BS75 * chore: refresh the sqlx cache for the repository-keyed credential queries Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01C1xHmkxuxYb1GYvth1BS75 * fix: gate the credential pass budget on the features that use it Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01C1xHmkxuxYb1GYvth1BS75 * fix: decide credential rotation ownership by repository, not resource path Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01C1xHmkxuxYb1GYvth1BS75 * refactor: renew only the credentials windmill holds, not tokens in a repo url Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01C1xHmkxuxYb1GYvth1BS75 * fix: authenticate the fork-branch poll and correct the renewal guidance Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01C1xHmkxuxYb1GYvth1BS75 * fix: do not claim a managed credential for a url the client cannot resolve Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01C1xHmkxuxYb1GYvth1BS75 * fix: define the credential facade for private builds without enterprise Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01C1xHmkxuxYb1GYvth1BS75 * fix: pin the listed token before the await and name the real renewal blocker Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01C1xHmkxuxYb1GYvth1BS75 * fix: pin the token the replace flow checked, and derive the scope test once Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01C1xHmkxuxYb1GYvth1BS75 * fix: classify the renewal state once so the card cannot contradict itself Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01C1xHmkxuxYb1GYvth1BS75 * refactor: ask only whether the token gets renewed, not why it does not Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01C1xHmkxuxYb1GYvth1BS75 * refactor: replace the managed-credential marker with a server answer Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01C1xHmkxuxYb1GYvth1BS75 * fix: read renewal from the credential and its origin, not a removed field Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01C1xHmkxuxYb1GYvth1BS75 * fix: read the provider for url-token repos, await the origin before defaulting, and visit unchecked repos last The maintenance pass sorted repositories with no recorded check first on the premise that they cost nothing, but a token-in-URL remote on a host that is not GitLab is probed every pass and never records a check, so it held the head of the list ahead of the tokens that expire. Such repositories now sort last. The card decided its delivery defaults before the origin lookup landed, so a freshly picked GitLab repository never got webhook delivery; the two lookups are awaited together. The resource editor offers to replace a token only where it is held, not in a fork that borrows it, and the replace flow refuses a URL it cannot parse instead of keying the token to it. Attaching a stored credential to a commit-hash probe now requires admin, matching the installation credential beside it. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01C1xHmkxuxYb1GYvth1BS75 * docs: describe the gitlab listing token the way the picker and the setup guide do Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01C1xHmkxuxYb1GYvth1BS75 * refactor: a token in the repository url is a plain remote, not a tracked credential Drops the status fingerprint that told one URL token from another, the docs' promise that such a token's expiry is reported, and the test's expectation that a URL-token repository declares a host. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01C1xHmkxuxYb1GYvth1BS75 * fix: the card reads the credential origin for managed controls and honours the licence for a borrowed token Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01C1xHmkxuxYb1GYvth1BS75 * chore: bump the ee ref Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01C1xHmkxuxYb1GYvth1BS75 * fix: hide a repository's credential line once nothing is held for the repository it names Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01C1xHmkxuxYb1GYvth1BS75 * docs: describe the exported credential status as it is Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01C1xHmkxuxYb1GYvth1BS75 * refactor: run the credential maintenance pass as its own task, without a budget The pass ran inside the monitor's join, whose deadline cancels every future in it, and a rotation cancelled between GitLab issuing a token and Windmill storing it loses the token family. A wall-clock budget with a least-recently-checked ordering kept it under the deadline. Spawning the pass instead makes the deadline irrelevant, so the budget, the ordering and the counter go; the advisory lock keeps a slow pass from overlapping the next, as it already did. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01C1xHmkxuxYb1GYvth1BS75 * docs: say what detaching the maintenance pass buys, and what it does not Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01C1xHmkxuxYb1GYvth1BS75 * chore: run git sync on the hub script version that reads a stored credential Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01C1xHmkxuxYb1GYvth1BS75 * chore: run the deploy push and the connection test on the hub versions that read a stored credential Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01C1xHmkxuxYb1GYvth1BS75 * fix: keep App repositories and plain remotes out of the stored-credential paths Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01C1xHmkxuxYb1GYvth1BS75 * chore: host-neutral deploy preview wording, drop the project filter from the GitLab picker Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01C1xHmkxuxYb1GYvth1BS75 * chore: bump ee ref, rotation no longer retains a second connection per repository Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01C1xHmkxuxYb1GYvth1BS75 * chore: bump ee ref, the rotation write-back holds a single connection Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01C1xHmkxuxYb1GYvth1BS75 * chore: hold the credential maintenance lock in a transaction so a dead sweep releases it Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01C1xHmkxuxYb1GYvth1BS75 * docs: describe the credential-stored callback as it fires Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01C1xHmkxuxYb1GYvth1BS75 * fix: keep the credential maintenance lock past the pool's idle-in-transaction timeout Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01C1xHmkxuxYb1GYvth1BS75 * chore: update ee-repo-ref to e092518ee60e33160fee9ae91a4d109566f7b0ee This commit updates the EE repository reference after PR #771 was merged in windmill-ee-private. Previous ee-repo-ref: 74481f7cc345757aebb2a8b04d3a22978328c348 New ee-repo-ref: e092518ee60e33160fee9ae91a4d109566f7b0ee Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Opus 5 Co-authored-by: windmill-internal-app[bot] --- ...83dce0c9986f3847227c2f66daa84d4109d7d.json | 15 ++ ...e72dc33ca421338e04277ac91c745351f89f3.json | 23 ++ ...1d3f9e80375b52748c458854a4883a6ecc8f6.json | 24 ++ ...31c79281d5cb2b0e81f1e832eac6eb3cc32e2.json | 16 ++ ...260d5ead34a6a3783dd21db08c7ee049eb232.json | 15 ++ ...a4d446b8ea00cf51e02138a06c245ab33252e.json | 16 ++ ...fd2a14b2e9e613bcf4bc80abbfd495637ab2f.json | 16 ++ ...6e96cd3a980411d80c6b10bae949cf2c42385.json | 22 ++ ...496c9448cffba5786b51e748bec0165a6519c.json | 17 ++ ...05d53bc756d00c8d28462097717502d82cc15.json | 23 ++ ...ced12d26f67f1b059c7c2afe66f8e4b3a9749.json | 22 ++ ...c90d7d82bd20ea30f1dc08f2e2066fb8d6f4b.json | 26 ++ ...18313f0434758413dbf83f8575d09de98fc25.json | 23 ++ backend/ee-repo-ref.txt | 2 +- ...it_credentials_workspace_settings.down.sql | 1 + ..._git_credentials_workspace_settings.up.sql | 8 + backend/src/monitor.rs | 189 ++++++++++++- .../fixtures/git_sync_fork_credential.sql | 55 ++++ backend/tests/git_sync_fork_credential.rs | 252 ++++++++++++++++++ .../windmill-api-workspaces/src/workspaces.rs | 38 ++- .../src/workspaces_extra.rs | 2 +- backend/windmill-api/openapi.yaml | 178 ++++++++++++- backend/windmill-api/src/git_sync_oss.rs | 5 + backend/windmill-api/src/lib.rs | 9 + backend/windmill-api/src/workspaces_export.rs | 16 +- backend/windmill-common/src/git_sync_oss.rs | 24 +- backend/windmill-common/src/workspaces.rs | 53 +++- backend/windmill-store/src/resources.rs | 19 ++ .../windmill-worker/src/result_processor.rs | 130 ++++++--- docs/git-sync-gitlab-setup.md | 170 ++++++++++++ .../src/lib/components/ApiConnectForm.svelte | 23 ++ .../src/lib/components/AppConnectInner.svelte | 9 + .../components/GitHubAppIntegration.svelte | 4 +- .../lib/components/GitLabIntegration.svelte | 221 +++++++++++++++ .../src/lib/components/ResourceEditor.svelte | 62 ++++- .../src/lib/components/ResourceForm.svelte | 27 +- .../git_sync/GitSyncRepositoryCard.svelte | 181 +++++++++++-- .../git_sync/ReplaceGitCredential.svelte | 134 ++++++++++ frontend/src/lib/hubPaths.json | 4 +- 39 files changed, 1975 insertions(+), 99 deletions(-) create mode 100644 backend/.sqlx/query-1d8ccd32266637d7f7915f92a8483dce0c9986f3847227c2f66daa84d4109d7d.json create mode 100644 backend/.sqlx/query-1fc9a0aeabb0a33efc37167be3de72dc33ca421338e04277ac91c745351f89f3.json create mode 100644 backend/.sqlx/query-48055203c97499ab4fc4dcbe9271d3f9e80375b52748c458854a4883a6ecc8f6.json create mode 100644 backend/.sqlx/query-64f72d7477f7c82d1596560ff6c31c79281d5cb2b0e81f1e832eac6eb3cc32e2.json create mode 100644 backend/.sqlx/query-73a3417f5331032a9b2afcf5b1c260d5ead34a6a3783dd21db08c7ee049eb232.json create mode 100644 backend/.sqlx/query-9c0ebee945eee03d667a4f38d18a4d446b8ea00cf51e02138a06c245ab33252e.json create mode 100644 backend/.sqlx/query-af5609185c6218d12fcb4a4234cfd2a14b2e9e613bcf4bc80abbfd495637ab2f.json create mode 100644 backend/.sqlx/query-b2f91eb32edd8db1605aeb5a17b6e96cd3a980411d80c6b10bae949cf2c42385.json create mode 100644 backend/.sqlx/query-bbb8331348216892e4714e4338c496c9448cffba5786b51e748bec0165a6519c.json create mode 100644 backend/.sqlx/query-ce3340a43a141cc7527211db16905d53bc756d00c8d28462097717502d82cc15.json create mode 100644 backend/.sqlx/query-d0eeb992a826d26376e9802a8c4ced12d26f67f1b059c7c2afe66f8e4b3a9749.json create mode 100644 backend/.sqlx/query-ddd41de79b23b0436bbc7997751c90d7d82bd20ea30f1dc08f2e2066fb8d6f4b.json create mode 100644 backend/.sqlx/query-f3a27f7781d986d9917a7022e1118313f0434758413dbf83f8575d09de98fc25.json create mode 100644 backend/migrations/20260904110943_git_credentials_workspace_settings.down.sql create mode 100644 backend/migrations/20260904110943_git_credentials_workspace_settings.up.sql create mode 100644 backend/tests/fixtures/git_sync_fork_credential.sql create mode 100644 backend/tests/git_sync_fork_credential.rs create mode 100644 docs/git-sync-gitlab-setup.md create mode 100644 frontend/src/lib/components/GitLabIntegration.svelte create mode 100644 frontend/src/lib/components/git_sync/ReplaceGitCredential.svelte diff --git a/backend/.sqlx/query-1d8ccd32266637d7f7915f92a8483dce0c9986f3847227c2f66daa84d4109d7d.json b/backend/.sqlx/query-1d8ccd32266637d7f7915f92a8483dce0c9986f3847227c2f66daa84d4109d7d.json new file mode 100644 index 0000000000..b9cbc6c382 --- /dev/null +++ b/backend/.sqlx/query-1d8ccd32266637d7f7915f92a8483dce0c9986f3847227c2f66daa84d4109d7d.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace_settings (workspace_id, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, git_credentials, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts, guest_access_enabled, guest_jwt_public_key, guest_jwt_jwks_url) SELECT $1, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, git_credentials, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts, guest_access_enabled, guest_jwt_public_key, guest_jwt_jwks_url FROM workspace_settings WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "1d8ccd32266637d7f7915f92a8483dce0c9986f3847227c2f66daa84d4109d7d" +} diff --git a/backend/.sqlx/query-1fc9a0aeabb0a33efc37167be3de72dc33ca421338e04277ac91c745351f89f3.json b/backend/.sqlx/query-1fc9a0aeabb0a33efc37167be3de72dc33ca421338e04277ac91c745351f89f3.json new file mode 100644 index 0000000000..6b4324af4c --- /dev/null +++ b/backend/.sqlx/query-1fc9a0aeabb0a33efc37167be3de72dc33ca421338e04277ac91c745351f89f3.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT (\n SELECT elem\n FROM jsonb_array_elements(git_credentials) AS elem\n WHERE elem->>'repo_identity' = $2\n )\n FROM workspace_settings WHERE workspace_id = $1 FOR UPDATE\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "elem", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "1fc9a0aeabb0a33efc37167be3de72dc33ca421338e04277ac91c745351f89f3" +} diff --git a/backend/.sqlx/query-48055203c97499ab4fc4dcbe9271d3f9e80375b52748c458854a4883a6ecc8f6.json b/backend/.sqlx/query-48055203c97499ab4fc4dcbe9271d3f9e80375b52748c458854a4883a6ecc8f6.json new file mode 100644 index 0000000000..b2cfb1cd89 --- /dev/null +++ b/backend/.sqlx/query-48055203c97499ab4fc4dcbe9271d3f9e80375b52748c458854a4883a6ecc8f6.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT elem->'credential'\n FROM workspace_settings, jsonb_array_elements(git_sync->'repositories') AS elem\n WHERE workspace_id = $1 AND elem->>'git_repo_resource_path' IN ($2, $3)\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "48055203c97499ab4fc4dcbe9271d3f9e80375b52748c458854a4883a6ecc8f6" +} diff --git a/backend/.sqlx/query-64f72d7477f7c82d1596560ff6c31c79281d5cb2b0e81f1e832eac6eb3cc32e2.json b/backend/.sqlx/query-64f72d7477f7c82d1596560ff6c31c79281d5cb2b0e81f1e832eac6eb3cc32e2.json new file mode 100644 index 0000000000..dc1f6ffbb1 --- /dev/null +++ b/backend/.sqlx/query-64f72d7477f7c82d1596560ff6c31c79281d5cb2b0e81f1e832eac6eb3cc32e2.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE workspace_settings\n SET git_credentials = (\n SELECT COALESCE(jsonb_agg(elem), '[]'::jsonb)\n FROM jsonb_array_elements(git_credentials) AS elem\n WHERE elem->>'repo_identity' IS DISTINCT FROM $2\n ) || jsonb_build_array($3::jsonb)\n WHERE workspace_id = $1\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "64f72d7477f7c82d1596560ff6c31c79281d5cb2b0e81f1e832eac6eb3cc32e2" +} diff --git a/backend/.sqlx/query-73a3417f5331032a9b2afcf5b1c260d5ead34a6a3783dd21db08c7ee049eb232.json b/backend/.sqlx/query-73a3417f5331032a9b2afcf5b1c260d5ead34a6a3783dd21db08c7ee049eb232.json new file mode 100644 index 0000000000..7c11dc80c0 --- /dev/null +++ b/backend/.sqlx/query-73a3417f5331032a9b2afcf5b1c260d5ead34a6a3783dd21db08c7ee049eb232.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace_settings (workspace_id, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, git_credentials, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts) SELECT $1, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, git_credentials, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts FROM workspace_settings WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "73a3417f5331032a9b2afcf5b1c260d5ead34a6a3783dd21db08c7ee049eb232" +} diff --git a/backend/.sqlx/query-9c0ebee945eee03d667a4f38d18a4d446b8ea00cf51e02138a06c245ab33252e.json b/backend/.sqlx/query-9c0ebee945eee03d667a4f38d18a4d446b8ea00cf51e02138a06c245ab33252e.json new file mode 100644 index 0000000000..6ef7b20a4f --- /dev/null +++ b/backend/.sqlx/query-9c0ebee945eee03d667a4f38d18a4d446b8ea00cf51e02138a06c245ab33252e.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE workspace_settings\n SET git_credentials = (\n SELECT COALESCE(jsonb_agg(elem), '[]'::jsonb)\n FROM jsonb_array_elements(git_credentials) AS elem\n WHERE elem->>'repo_identity' IS DISTINCT FROM $2\n ) || jsonb_build_array($3::jsonb)\n WHERE workspace_id = $1\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "9c0ebee945eee03d667a4f38d18a4d446b8ea00cf51e02138a06c245ab33252e" +} diff --git a/backend/.sqlx/query-af5609185c6218d12fcb4a4234cfd2a14b2e9e613bcf4bc80abbfd495637ab2f.json b/backend/.sqlx/query-af5609185c6218d12fcb4a4234cfd2a14b2e9e613bcf4bc80abbfd495637ab2f.json new file mode 100644 index 0000000000..41481e2ec7 --- /dev/null +++ b/backend/.sqlx/query-af5609185c6218d12fcb4a4234cfd2a14b2e9e613bcf4bc80abbfd495637ab2f.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE workspace_settings\n SET git_credentials = (\n SELECT COALESCE(jsonb_agg(elem), '[]'::jsonb)\n FROM jsonb_array_elements(git_credentials) AS elem\n WHERE elem->>'repo_identity' IS DISTINCT FROM $2\n ) || jsonb_build_array($3::jsonb)\n WHERE workspace_id = $1\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "af5609185c6218d12fcb4a4234cfd2a14b2e9e613bcf4bc80abbfd495637ab2f" +} diff --git a/backend/.sqlx/query-b2f91eb32edd8db1605aeb5a17b6e96cd3a980411d80c6b10bae949cf2c42385.json b/backend/.sqlx/query-b2f91eb32edd8db1605aeb5a17b6e96cd3a980411d80c6b10bae949cf2c42385.json new file mode 100644 index 0000000000..222bfc0f82 --- /dev/null +++ b/backend/.sqlx/query-b2f91eb32edd8db1605aeb5a17b6e96cd3a980411d80c6b10bae949cf2c42385.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT pg_advisory_unlock(hashtext($1)::bigint)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "pg_advisory_unlock", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "b2f91eb32edd8db1605aeb5a17b6e96cd3a980411d80c6b10bae949cf2c42385" +} diff --git a/backend/.sqlx/query-bbb8331348216892e4714e4338c496c9448cffba5786b51e748bec0165a6519c.json b/backend/.sqlx/query-bbb8331348216892e4714e4338c496c9448cffba5786b51e748bec0165a6519c.json new file mode 100644 index 0000000000..980ba7a36c --- /dev/null +++ b/backend/.sqlx/query-bbb8331348216892e4714e4338c496c9448cffba5786b51e748bec0165a6519c.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE workspace_settings\n SET git_sync = jsonb_set(git_sync, '{repositories}',\n COALESCE((SELECT jsonb_agg(\n CASE WHEN elem->>'git_repo_resource_path' IN ($2, $3)\n THEN CASE WHEN $4::jsonb = 'null'::jsonb\n THEN elem - 'credential'\n ELSE jsonb_set(elem, '{credential}', $4) END\n ELSE elem END)\n FROM jsonb_array_elements(git_sync->'repositories') AS elem), '[]'::jsonb)\n )\n WHERE workspace_id = $1\n AND jsonb_typeof(git_sync->'repositories') = 'array'\n AND EXISTS (\n SELECT 1 FROM jsonb_array_elements(git_sync->'repositories') AS e\n WHERE e->>'git_repo_resource_path' IN ($2, $3)\n )\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "bbb8331348216892e4714e4338c496c9448cffba5786b51e748bec0165a6519c" +} diff --git a/backend/.sqlx/query-ce3340a43a141cc7527211db16905d53bc756d00c8d28462097717502d82cc15.json b/backend/.sqlx/query-ce3340a43a141cc7527211db16905d53bc756d00c8d28462097717502d82cc15.json new file mode 100644 index 0000000000..58300795af --- /dev/null +++ b/backend/.sqlx/query-ce3340a43a141cc7527211db16905d53bc756d00c8d28462097717502d82cc15.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT elem\n FROM workspace_settings, jsonb_array_elements(git_credentials) AS elem\n WHERE workspace_id = $1 AND elem->>'repo_identity' = $2\n LIMIT 1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "elem", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "ce3340a43a141cc7527211db16905d53bc756d00c8d28462097717502d82cc15" +} diff --git a/backend/.sqlx/query-d0eeb992a826d26376e9802a8c4ced12d26f67f1b059c7c2afe66f8e4b3a9749.json b/backend/.sqlx/query-d0eeb992a826d26376e9802a8c4ced12d26f67f1b059c7c2afe66f8e4b3a9749.json new file mode 100644 index 0000000000..cbcb7f18f7 --- /dev/null +++ b/backend/.sqlx/query-d0eeb992a826d26376e9802a8c4ced12d26f67f1b059c7c2afe66f8e4b3a9749.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT pg_try_advisory_lock(hashtext($1)::bigint)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "pg_try_advisory_lock", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "d0eeb992a826d26376e9802a8c4ced12d26f67f1b059c7c2afe66f8e4b3a9749" +} diff --git a/backend/.sqlx/query-ddd41de79b23b0436bbc7997751c90d7d82bd20ea30f1dc08f2e2066fb8d6f4b.json b/backend/.sqlx/query-ddd41de79b23b0436bbc7997751c90d7d82bd20ea30f1dc08f2e2066fb8d6f4b.json new file mode 100644 index 0000000000..9622e63a56 --- /dev/null +++ b/backend/.sqlx/query-ddd41de79b23b0436bbc7997751c90d7d82bd20ea30f1dc08f2e2066fb8d6f4b.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT ws.workspace_id, ws.git_sync\n FROM workspace_settings ws\n JOIN workspace w ON w.id = ws.workspace_id\n WHERE NOT w.deleted\n AND ws.git_sync IS NOT NULL\n AND jsonb_typeof(ws.git_sync->'repositories') = 'array'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "git_sync", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + true + ] + }, + "hash": "ddd41de79b23b0436bbc7997751c90d7d82bd20ea30f1dc08f2e2066fb8d6f4b" +} \ No newline at end of file diff --git a/backend/.sqlx/query-f3a27f7781d986d9917a7022e1118313f0434758413dbf83f8575d09de98fc25.json b/backend/.sqlx/query-f3a27f7781d986d9917a7022e1118313f0434758413dbf83f8575d09de98fc25.json new file mode 100644 index 0000000000..1a71c74e9e --- /dev/null +++ b/backend/.sqlx/query-f3a27f7781d986d9917a7022e1118313f0434758413dbf83f8575d09de98fc25.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT (\n SELECT elem\n FROM jsonb_array_elements(git_credentials) AS elem\n WHERE elem->>'repo_identity' = $2\n )\n FROM workspace_settings WHERE workspace_id = $1 FOR UPDATE\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "elem", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "f3a27f7781d986d9917a7022e1118313f0434758413dbf83f8575d09de98fc25" +} diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 0ebbb1b5ea..8340c8d90e 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -d33ea730c550cdbc7d050aeb6d40dcef3d134e07 +e092518ee60e33160fee9ae91a4d109566f7b0ee diff --git a/backend/migrations/20260904110943_git_credentials_workspace_settings.down.sql b/backend/migrations/20260904110943_git_credentials_workspace_settings.down.sql new file mode 100644 index 0000000000..ca88ec0f1e --- /dev/null +++ b/backend/migrations/20260904110943_git_credentials_workspace_settings.down.sql @@ -0,0 +1 @@ +ALTER TABLE workspace_settings DROP COLUMN IF EXISTS git_credentials; diff --git a/backend/migrations/20260904110943_git_credentials_workspace_settings.up.sql b/backend/migrations/20260904110943_git_credentials_workspace_settings.up.sql new file mode 100644 index 0000000000..7ece575cc1 --- /dev/null +++ b/backend/migrations/20260904110943_git_credentials_workspace_settings.up.sql @@ -0,0 +1,8 @@ +-- Server-owned git-sync credentials, one entry per repository the workspace +-- holds a token for, keyed by that repository rather than by a resource naming +-- it: a resource's URL is writable, and its path is not settled while it is +-- being created. +-- Kept out of `git_sync` because that column is copied into forks and returned +-- by the workspace settings API; this one is copied by neither. +ALTER TABLE workspace_settings + ADD COLUMN IF NOT EXISTS git_credentials JSONB NOT NULL DEFAULT '[]'::jsonb; diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index a11ed55882..9aae2cca06 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -354,7 +354,9 @@ pub async fn initial_load( ) } }); - pass.action(windmill_common::min_version::store_min_keep_alive_version(db)); + pass.action(windmill_common::min_version::store_min_keep_alive_version( + db, + )); pass.setting( windmill_common::global_settings::INSTANCE_EVENTS_WEBHOOK_SETTING, false, @@ -709,7 +711,6 @@ pub async fn initial_load( pass.run(conn).await; } - pub fn apply_metrics_enabled(value: Option) { if let Some(serde_json::Value::Bool(t)) = value { METRICS_ENABLED.store(t, Ordering::Relaxed) @@ -1066,8 +1067,8 @@ pub fn apply_fork_workspace_tag_append_fork_suffix(value: Option error::Result<()> { - let v = - load_value_from_global_settings_with_conn(conn, CRITICAL_ALERT_MUTE_UI_SETTING, true).await?; + let v = load_value_from_global_settings_with_conn(conn, CRITICAL_ALERT_MUTE_UI_SETTING, true) + .await?; apply_critical_alert_mute_ui_setting(v); Ok(()) } @@ -2732,7 +2733,6 @@ pub async fn reload_timeout_wait_result_setting(conn: &Connection) { .await; } - pub async fn reload_extra_pip_index_url_setting(conn: &Connection) { reload_option_setting_with_tracing( conn, @@ -2823,7 +2823,6 @@ pub async fn reload_bunfig_install_scopes_setting(conn: &Connection) { .await; } - pub async fn reload_nuget_config_setting(conn: &Connection) { reload_option_setting_with_tracing( conn, @@ -2931,7 +2930,6 @@ pub async fn reload_ruby_repos_setting(conn: &Connection) { .await; } - pub async fn reload_workspace_registries_setting(conn: &Connection) { match load_value_from_global_settings_with_conn( conn, @@ -3185,7 +3183,6 @@ pub async fn apply_job_isolation_setting(value: Option) { } } - async fn resolve_license_key_value(conn: &Connection, quiet: bool) -> anyhow::Result { let q = load_value_from_global_settings_with_conn(conn, LICENSE_KEY_SETTING, true) .await @@ -3480,7 +3477,10 @@ impl<'a> SettingsPass<'a> { // on compile-time defaults until the next full reload. Only the single-query transport // can fail this way; over HTTP the batch already is the per-setting read. if matches!(conn, Connection::Sql(_)) && values.is_empty() && !names.is_empty() { - tracing::warn!("Falling back to per-setting reads for {} settings", names.len()); + tracing::warn!( + "Falling back to per-setting reads for {} settings", + names.len() + ); values = fetch_settings_individually(conn, &names).await; } for (name, http) in &declared { @@ -3872,7 +3872,6 @@ pub fn parse_setting_value( value } - #[cfg(feature = "prometheus")] pub async fn monitor_pool(db: &DB) { if METRICS_ENABLED.load(Ordering::Relaxed) { @@ -4316,6 +4315,24 @@ pub async fn monitor_db( } }; + // Re-check what each git-sync repository's own credential says about its expiry, + // and rotate the ones close to it. Every ~40 min: the values move over days, and + // `should_run` counts iterations in a u8. Spawned rather than joined: the join + // below is cancelled at its deadline, which a long sweep would reach, and a + // rotation cut off between GitLab issuing a token and Windmill storing it + // loses the token family. Detached, only process shutdown can cut it off, + // which a rotation almost never coincides with. The pass's advisory lock + // keeps a slow one from overlapping the next. + let git_credential_maintenance_f = async { + #[cfg(all(feature = "enterprise", feature = "private"))] + if server_mode && iteration.is_some() && iteration.as_ref().unwrap().should_run(240) { + if let Some(db) = conn.as_sql() { + let db = db.clone(); + tokio::spawn(async move { maintain_git_credentials(&db).await }); + } + } + }; + // run every 2 iterations (~20s at the default LISTEN_NEW_EVENTS_INTERVAL_SEC). // Enterprise feature: the active `// freshness` backstop lives in // windmill-queue's `freshness_watchdog` (`private`); OSS gets a no-op stub. @@ -4369,6 +4386,7 @@ pub async fn monitor_db( export_audit_logs_to_object_store_f, cleanup_scheduled_job_deletions_f, git_auto_pull_f, + git_credential_maintenance_f, pipeline_freshness_watchdog_f, reconcile_unarmed_schedules_f, ); @@ -4691,6 +4709,156 @@ lazy_static::lazy_static! { #[cfg(feature = "private")] const AUTO_PULL_POLL_SLACK_S: i64 = 30; +/// Advisory lock id ensuring only one server replica maintains git credentials at +/// a time (adjacent to GIT_AUTO_PULL_LOCK_ID). +#[cfg(all(feature = "enterprise", feature = "private"))] +const GIT_CREDENTIAL_LOCK_ID: i64 = 737_483_923; + +/// Refresh every git-sync repository's credential status and rotate the ones near +/// expiry, so a token dies visibly (and usually not at all) rather than taking +/// sync down on its expiry date. +#[cfg(all(feature = "enterprise", feature = "private"))] +async fn maintain_git_credentials(db: &Pool) { + use windmill_common::ee_oss::{get_license_plan, LicensePlan}; + + if !matches!(get_license_plan().await, LicensePlan::Enterprise) { + return; + } + + // Transaction-scoped advisory lock, as for the schedule reconcile above: a + // session lock on a pooled connection would ride back into the pool still + // held if the sweep died before unlocking, and wedge the pass on every + // replica until a restart. The transaction only owns the lock; the sweep + // commits each status and each rotated token on its own as it goes. + let mut lock_tx = match db.begin().await { + Ok(tx) => tx, + Err(e) => { + tracing::error!("git credentials: failed to begin lock tx: {e:#}"); + return; + } + }; + // The transaction stays idle while the sweep talks to git hosts, and the + // pool's ten-minute idle-in-transaction timeout would end it, lock included, + // partway through a sweep over enough slow hosts. Lifted for this + // transaction only; it dies with the connection either way. + if let Err(e) = sqlx::query("SET LOCAL idle_in_transaction_session_timeout = 0") + .execute(&mut *lock_tx) + .await + { + tracing::error!("git credentials: failed to lift the idle timeout: {e:#}"); + return; + } + let locked: bool = match sqlx::query_scalar("SELECT pg_try_advisory_xact_lock($1)") + .bind(GIT_CREDENTIAL_LOCK_ID) + .fetch_one(&mut *lock_tx) + .await + { + Ok(v) => v, + Err(e) => { + tracing::error!("git credentials: advisory lock failed: {e:#}"); + return; + } + }; + if !locked { + return; + } + + if let Err(e) = maintain_git_credentials_inner(db).await { + tracing::error!("git credentials: maintenance error: {e:#}"); + } + drop(lock_tx); +} + +#[cfg(all(feature = "enterprise", feature = "private"))] +async fn maintain_git_credentials_inner(db: &Pool) -> error::Result<()> { + use windmill_common::workspaces::WorkspaceGitSyncSettings; + + // Same deleted/archived exclusion as the auto-pull poller: a dead workspace's + // settings row survives, and rotating a token for one would be pure damage. + let rows = sqlx::query!( + r#"SELECT ws.workspace_id, ws.git_sync + FROM workspace_settings ws + JOIN workspace w ON w.id = ws.workspace_id + WHERE NOT w.deleted + AND ws.git_sync IS NOT NULL + AND jsonb_typeof(ws.git_sync->'repositories') = 'array'"# + ) + .fetch_all(db) + .await?; + + for row in rows { + let Some(git_sync) = row.git_sync else { + continue; + }; + let settings: WorkspaceGitSyncSettings = match serde_json::from_value(git_sync) { + Ok(s) => s, + Err(e) => { + tracing::warn!( + "git credentials: invalid git_sync settings for workspace {}: {e}", + row.workspace_id + ); + continue; + } + }; + + for repo in settings.repositories.iter() { + let path = &repo.git_repo_resource_path; + // This refreshes and records the status on every repository it looks at, + // rotating only the ones near expiry, so it is the whole maintenance pass + // rather than just the rotation half. + if let Err(e) = windmill_common::git_sync_ee::rotate_git_credential_if_due( + db, + &row.workspace_id, + path, + ) + .await + { + tracing::error!( + "git credentials: maintenance failed for {path} in workspace {}: {e:#}", + row.workspace_id + ); + } + + // A repository that wants webhook delivery but holds no hook never + // gets one otherwise: the reconcile runs on a settings save, so a + // credential that was unusable when the hook should have been created + // would leave it missing until an admin saved again. Checking stored + // state costs nothing, and only the repositories actually missing a + // hook reach the host. + use windmill_common::workspaces::AutoPullMode; + // Also when a hook exists but carries a warning: a save during a GitLab + // outage keeps the hook and records why it could not be confirmed, and + // that warning is only cleared by a reconcile that confirms it again. + // Only repositories with a checked credential of their own: for + // everything else a settings save stays the one place hooks are + // reconciled, so an App repository or a plain remote is never touched + // here, and a recorded delivery mode nobody saved is never normalized. + let needs_hook = repo.credential.is_some() + && repo.auto_pull.as_ref().is_some_and(|a| { + a.enabled + && matches!(a.mode, AutoPullMode::Auto | AutoPullMode::Webhook) + && (a.webhook_id.is_none() || a.webhook_error.is_some()) + }); + if needs_hook { + let mut repo = repo.clone(); + if let Err(e) = windmill_common::git_sync_ee::sync_repo_webhook( + db, + &row.workspace_id, + &mut repo, + ) + .await + { + tracing::warn!( + "git credentials: could not reconcile the webhook for {path} in workspace {}: {e:#}", + row.workspace_id + ); + } + } + } + } + Ok(()) +} + #[cfg(feature = "private")] async fn poll_git_auto_pull_inner(db: &Pool) -> error::Result<()> { use windmill_common::workspaces::{AutoPullMode, WorkspaceGitSyncSettings}; @@ -6588,7 +6756,6 @@ pub async fn reload_critical_alerts_on_db_oversize(conn: &DB) -> error::Result<( Ok(()) } - pub async fn reload_jwt_secret_setting(db: &DB) -> error::Result<()> { let v = load_value_from_global_settings(db, JWT_SECRET_SETTING).await?; apply_jwt_secret_setting(db, v).await diff --git a/backend/tests/fixtures/git_sync_fork_credential.sql b/backend/tests/fixtures/git_sync_fork_credential.sql new file mode 100644 index 0000000000..1c1d8bd1e1 --- /dev/null +++ b/backend/tests/fixtures/git_sync_fork_credential.sql @@ -0,0 +1,55 @@ +-- A parent whose git-sync repository has a recorded credential, and the workspace +-- shapes the credential lookup and the qualification predicate have to tell apart. +-- +-- The credential itself is shared down the fork chain; the recorded *status* is +-- not, because it describes one repository and a fork can repoint its copy of the +-- resource. Forks get a status by fork creation copying it, which no fixture here +-- simulates, so a fork without one is a workspace nothing has checked yet. + +INSERT INTO workspace (id, name, owner, parent_workspace_id) VALUES + ('parent-ws', 'parent-ws', 'test-user', NULL), + ('fork-ws', 'fork-ws', 'test-user', 'parent-ws'), + -- A fork of a fork: the shape a fork of a dev workspace takes, and the one a + -- parent-only lookup misses. + ('deep-fork-ws', 'deep-fork-ws', 'test-user', 'fork-ws'), + ('errored-fork-ws', 'errored-fork-ws', 'test-user', 'parent-ws'), + ('orphan-ws', 'orphan-ws', 'test-user', NULL); + +-- A stored credential is encrypted with its own workspace's key. +INSERT INTO workspace_key (workspace_id, kind, key) VALUES + ('parent-ws', 'cloud', 'parent-key'), + ('fork-ws', 'cloud', 'fork-key'), + ('deep-fork-ws', 'cloud', 'deep-fork-key'), + ('errored-fork-ws', 'cloud', 'errored-fork-key'), + ('orphan-ws', 'cloud', 'orphan-key'); + +-- The parent holds the credential. +INSERT INTO workspace_settings (workspace_id, git_sync) VALUES + ('parent-ws', '{"repositories":[{"git_repo_resource_path":"$res:u/admin/repo", + "credential":{"provider":"gitlab","rotatable":true,"checked_at":1788500000}}]}'), + +-- A fork inherits the repository but not the credential: this is what +-- clone_workspace_data leaves behind. + ('fork-ws', '{"repositories":[{"git_repo_resource_path":"$res:u/admin/repo"}]}'), + +-- Two levels down, so neither the credential nor its status is one hop away. + ('deep-fork-ws', '{"repositories":[{"git_repo_resource_path":"$res:u/admin/repo"}]}'), + +-- A fork whose own credential has since failed. Its own standing must win over +-- the parent's healthy record rather than being papered over. + ('errored-fork-ws', '{"repositories":[{"git_repo_resource_path":"$res:u/admin/repo", + "credential":{"provider":"gitlab","rotatable":false,"checked_at":1788500000, + "error":"GitLab no longer accepts this token"}}]}'), + +-- No credential and no parent to borrow one from. + ('orphan-ws', '{"repositories":[{"git_repo_resource_path":"$res:u/admin/repo"}]}'); + +-- The resource each repository entry names, all pointing at the same repository. +-- The errored fork carries its token in the URL, the way a repository configured +-- by hand does: a plain remote, whatever the chain above it holds. +INSERT INTO resource (workspace_id, path, value, resource_type) VALUES + ('parent-ws', 'u/admin/repo', '{"url":"https://gitlab.com/grp/proj.git"}', 'git_repository'), + ('fork-ws', 'u/admin/repo', '{"url":"https://gitlab.com/grp/proj.git"}', 'git_repository'), + ('deep-fork-ws', 'u/admin/repo', '{"url":"https://gitlab.com/grp/proj.git"}', 'git_repository'), + ('errored-fork-ws', 'u/admin/repo', '{"url":"https://oauth2:glpat-inline@gitlab.com/grp/proj.git"}', 'git_repository'), + ('orphan-ws', 'u/admin/repo', '{"url":"https://gitlab.com/grp/proj.git"}', 'git_repository'); diff --git a/backend/tests/git_sync_fork_credential.rs b/backend/tests/git_sync_fork_credential.rs new file mode 100644 index 0000000000..7e31136727 --- /dev/null +++ b/backend/tests/git_sync_fork_credential.rs @@ -0,0 +1,252 @@ +//! A fork reaches the git credential held above it in its fork chain. +//! +//! Fork creation copies the parent's git-sync repositories but not the credential, +//! which is stored per workspace so that rotation has one owner. Chains nest (a +//! fork of a dev workspace, a fork of that), so the depth-2 cases here are what +//! keep the lookup from regressing to the parent. +//! +//! The recorded *status* is not shared the same way: it describes one repository, +//! and a fork can repoint its copy of the resource, so each workspace answers from +//! its own record and gets one by fork creation copying it down. +#![cfg(all(feature = "enterprise", feature = "private"))] + +use sqlx::{Pool, Postgres}; +use windmill_common::git_sync_ee::{ + git_credential_for_url, repo_provider, repo_supports_managed_git_features, set_git_credential, + GitProvider, +}; +use windmill_common::workspaces::GitCredentialProvider; + +const REPO: &str = "$res:u/admin/repo"; +const URL: &str = "https://gitlab.com/grp/proj.git"; + +/// A repository is managed when a credential is held for the repository its +/// URL names now and the last check found it healthy. The recorded status is +/// keyed by resource path, so alone it would outlive a repoint; the held +/// credential alone says nothing about whether the host still accepts it. +#[sqlx::test(fixtures("git_sync_fork_credential"))] +async fn credential_status_is_a_workspaces_own(db: Pool) -> anyhow::Result<()> { + assert!( + !repo_supports_managed_git_features(&db, "parent-ws", REPO).await, + "a healthy status with nothing held behind it does not qualify" + ); + set_git_credential( + &db, + "parent-ws", + URL, + "glpat-secret", + GitCredentialProvider::Gitlab, + ) + .await?; + assert!( + repo_supports_managed_git_features(&db, "parent-ws", REPO).await, + "the workspace holding both the credential and the recorded status qualifies" + ); + assert!( + !repo_supports_managed_git_features(&db, "fork-ws", REPO).await, + "a fork borrowing the credential with no record of its own does not: the \ + status describes one repository, and this fork's resource could name another" + ); + assert!( + !repo_supports_managed_git_features(&db, "errored-fork-ws", REPO).await, + "a workspace whose own credential failed stays disqualified" + ); + Ok(()) +} + +/// The host a repository talks to is declared when its credential is stored, and +/// travels with the credential down the fork chain. +/// +/// Read from the recorded status instead, a fork answered with the default +/// provider until its own check ran, which is long enough to register a webhook +/// against the wrong receiver. +#[sqlx::test(fixtures("git_sync_fork_credential"))] +async fn the_provider_comes_from_the_credential_and_reaches_forks( + db: Pool, +) -> anyhow::Result<()> { + assert_eq!( + repo_provider(&db, "parent-ws", REPO).await, + GitProvider::GitHub, + "with nothing stored there is no declaration to read, so the default stands" + ); + + set_git_credential( + &db, + "parent-ws", + URL, + "glpat-secret", + GitCredentialProvider::Gitlab, + ) + .await?; + + assert_eq!( + repo_provider(&db, "parent-ws", REPO).await, + GitProvider::GitLab, + "the workspace that stored it reads its own declaration" + ); + assert_eq!( + repo_provider(&db, "fork-ws", REPO).await, + GitProvider::GitLab, + "and a fork resolving that credential reads it too, without a check of its own" + ); + assert_eq!( + repo_provider(&db, "deep-fork-ws", REPO).await, + GitProvider::GitLab, + "two levels down as well" + ); + assert_eq!( + repo_provider(&db, "orphan-ws", REPO).await, + GitProvider::GitHub, + "a workspace outside the chain resolves no credential and no declaration" + ); + assert_eq!( + repo_provider(&db, "errored-fork-ws", REPO).await, + GitProvider::GitHub, + "a token written into the URL makes the repository a plain remote: the \ + parent's credential is not consulted and no host is declared" + ); + Ok(()) +} + +/// The stored credential is shared with forks and keyed by one repository. +/// +/// Both properties are the point of keeping it in `workspace_settings` under the +/// repository's identity: sharing is what stops a rotation from stranding every +/// fork on a revoked token, and the key is what stops a rewritten resource URL +/// from carrying the token to a host of the writer's choosing. +#[sqlx::test(fixtures("git_sync_fork_credential"))] +async fn a_fork_reads_an_ancestors_credential_for_the_bound_repository_only( + db: Pool, +) -> anyhow::Result<()> { + set_git_credential( + &db, + "parent-ws", + URL, + "glpat-secret", + GitCredentialProvider::Gitlab, + ) + .await?; + + assert_eq!( + git_credential_for_url(&db, "parent-ws", URL) + .await? + .as_deref(), + Some("glpat-secret"), + "the workspace that stored it reads it back" + ); + assert_eq!( + git_credential_for_url(&db, "fork-ws", URL) + .await? + .as_deref(), + Some("glpat-secret"), + "a fork stores none of its own and resolves the parent's" + ); + assert_eq!( + git_credential_for_url(&db, "deep-fork-ws", URL) + .await? + .as_deref(), + Some("glpat-secret"), + "a fork of a fork resolves the root's, two levels up" + ); + assert_eq!( + git_credential_for_url(&db, "fork-ws", "https://evil.example/grp/proj.git").await?, + None, + "a resource repointed at another repository asks for that one's \ + credential and finds none" + ); + assert_eq!( + git_credential_for_url(&db, "orphan-ws", URL).await?, + None, + "a workspace with no credential and no parent resolves nothing" + ); + Ok(()) +} + +/// One repository's credential is untouched by another's. +/// +/// The key is the repository, so picking a second repository stores beside the +/// first rather than over it. Keyed by the resource instead, a workspace editing +/// one repository's resource to point somewhere else would replace the token the +/// original repository was still syncing with. +#[sqlx::test(fixtures("git_sync_fork_credential"))] +async fn each_repository_keeps_its_own_credential(db: Pool) -> anyhow::Result<()> { + const OTHER_URL: &str = "https://gitlab.com/grp/other.git"; + + set_git_credential( + &db, + "parent-ws", + URL, + "glpat-first", + GitCredentialProvider::Gitlab, + ) + .await?; + set_git_credential( + &db, + "parent-ws", + OTHER_URL, + "glpat-second", + GitCredentialProvider::Gitlab, + ) + .await?; + + assert_eq!( + git_credential_for_url(&db, "parent-ws", URL) + .await? + .as_deref(), + Some("glpat-first"), + "storing a second repository's token leaves the first's in place" + ); + assert_eq!( + git_credential_for_url(&db, "parent-ws", OTHER_URL) + .await? + .as_deref(), + Some("glpat-second") + ); + + set_git_credential( + &db, + "parent-ws", + URL, + "glpat-replacement", + GitCredentialProvider::Gitlab, + ) + .await?; + assert_eq!( + git_credential_for_url(&db, "parent-ws", URL) + .await? + .as_deref(), + Some("glpat-replacement"), + "storing the same repository again replaces rather than duplicates" + ); + assert_eq!( + git_credential_for_url(&db, "parent-ws", OTHER_URL) + .await? + .as_deref(), + Some("glpat-second"), + "and still leaves the other repository alone" + ); + Ok(()) +} + +/// A credential issued for `https` is not served for the `http` spelling. +/// +/// The resource holding the URL is writable by anyone with write on its path, so +/// without the scheme in the key that edit would send the token over cleartext. +#[sqlx::test(fixtures("git_sync_fork_credential"))] +async fn a_credential_is_not_served_over_a_downgraded_transport( + db: Pool, +) -> anyhow::Result<()> { + set_git_credential( + &db, + "parent-ws", + URL, + "glpat-secret", + GitCredentialProvider::Gitlab, + ) + .await?; + assert_eq!( + git_credential_for_url(&db, "parent-ws", "http://gitlab.com/grp/proj.git").await?, + None + ); + Ok(()) +} diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 93e0ad5ca8..85226ddf2c 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -3986,6 +3986,7 @@ async fn edit_git_sync_config( clear_client_supplied_auto_pull_state(ap); } repo.open_pr_error = None; + repo.credential = None; } reject_parent_only_git_sync_settings_on_fork( &db, @@ -4086,6 +4087,7 @@ async fn edit_git_sync_config( continue; }; repo.open_pr_error = old.open_pr_error.clone(); + repo.credential = old.credential.clone(); if let (Some(new_ap), Some(old_ap)) = (repo.auto_pull.as_mut(), old.auto_pull.as_ref()) { @@ -4131,6 +4133,7 @@ async fn edit_git_sync_config( .flatten() .and_then(|v| serde_json::from_value(v).ok()); let removed_webhooks: Vec<(String, i64)> = existing + .as_ref() .map(|e| { e.repositories .iter() @@ -4164,6 +4167,19 @@ async fn edit_git_sync_config( // `sync_repo_webhook` writes back the webhook fields it changes itself: // the remote hook and the record of it have to move together, so // persisting them out here would let one land without the other. + // Before the webhook reconcile, which decides whether this repo can have + // one from the credential this records. Also puts a short-lived or + // under-scoped token in front of the operator while they are still on the + // settings page, rather than when it expires. + if let Err(e) = windmill_common::git_sync_ee::refresh_git_credential_status( + &db, + &w_id, + &repo.git_repo_resource_path, + ) + .await + { + tracing::warn!("git credential check error: {}", e); + } if let Err(e) = windmill_common::git_sync_ee::sync_repo_webhook(&db, &w_id, repo).await { tracing::warn!("git auto-pull: webhook sync error: {}", e); @@ -4217,6 +4233,7 @@ async fn edit_git_sync_repository( clear_client_supplied_auto_pull_state(ap); } new_config.repository.open_pr_error = None; + new_config.repository.credential = None; reject_parent_only_git_sync_settings_on_fork( &db, &w_id, @@ -4341,6 +4358,7 @@ async fn edit_git_sync_repository( // from the UI cannot revert what the poller/webhook layer wrote. let mut updated = new_config.repository; updated.open_pr_error = existing_repo.open_pr_error.clone(); + updated.credential = existing_repo.credential.clone(); match (updated.auto_pull.as_mut(), existing_repo.auto_pull.as_ref()) { (Some(new_ap), Some(old_ap)) => { new_ap.last_synced_sha = old_ap.last_synced_sha.clone(); @@ -4398,6 +4416,15 @@ async fn edit_git_sync_repository( .iter_mut() .find(|r| r.git_repo_resource_path == new_config.git_repo_resource_path) { + if let Err(e) = windmill_common::git_sync_ee::refresh_git_credential_status( + &db, + &w_id, + &repo.git_repo_resource_path, + ) + .await + { + tracing::warn!("git credential check error: {}", e); + } if let Err(e) = windmill_common::git_sync_ee::sync_repo_webhook(&db, &w_id, repo).await { tracing::warn!("git auto-pull: webhook sync error: {}", e); } @@ -4537,6 +4564,10 @@ async fn delete_git_sync_repository( } } + // The stored credential is deliberately left alone: it belongs to the git + // repository resource, which this endpoint does not delete, and the resource + // still authenticates with it for connection tests and commit lookups. + // Trigger git sync for repository deletion handle_deployment_metadata( &authed.email, @@ -6306,9 +6337,10 @@ async fn update_workspace_settings( // Auto-pull and fork PRs are parent-owned and must not be inherited: // the fork would otherwise carry the parent's webhook id (turning off // auto-pull on the fork would delete the parent's webhook). A fork - // still inherits the push-direction config and the installation. - // Repo → fork sync is driven by the parent's webhook/poller - // (`sync_forks`), which routes the fork's `wm-fork/**` branch into it. + // still inherits the push-direction config, the installation, and the + // recorded credential status, which describes the repository rather + // than belonging to either workspace and would otherwise leave the + // fork unqualified for managed features until its first check. r.auto_pull = None; r.fork_open_prs = false; r.open_pr_error = None; diff --git a/backend/windmill-api-workspaces/src/workspaces_extra.rs b/backend/windmill-api-workspaces/src/workspaces_extra.rs index b2883e5f9a..e75f0d1ef2 100644 --- a/backend/windmill-api-workspaces/src/workspaces_extra.rs +++ b/backend/windmill-api-workspaces/src/workspaces_extra.rs @@ -113,7 +113,7 @@ pub(crate) async fn change_workspace_id( // Duplicate workspace settings (keep copy in old workspace for reference) info!("Duplicating workspace_settings table"); sqlx::query!( - "INSERT INTO workspace_settings (workspace_id, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts, guest_access_enabled, guest_jwt_public_key, guest_jwt_jwks_url) SELECT $1, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts, guest_access_enabled, guest_jwt_public_key, guest_jwt_jwks_url FROM workspace_settings WHERE workspace_id = $2", + "INSERT INTO workspace_settings (workspace_id, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, git_credentials, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts, guest_access_enabled, guest_jwt_public_key, guest_jwt_jwks_url) SELECT $1, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, git_credentials, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts, guest_access_enabled, guest_jwt_public_key, guest_jwt_jwks_url FROM workspace_settings WHERE workspace_id = $2", &rw.new_id, &old_id ) diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 4abd5f4dff..65a29c141d 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -2741,7 +2741,7 @@ paths: /w/{workspace}/github_app/token: post: - summary: get github app token + summary: get the git credential for a git-sync job (GitHub App token, or the credential stored for the repository) operationId: getGithubAppToken tags: - workspace @@ -2761,7 +2761,7 @@ paths: - job_token responses: "200": - description: github app token + description: git credential content: application/json: schema: @@ -2905,6 +2905,131 @@ paths: "200": description: Successfully imported the installation + /w/{workspace}/git_sync/gitlab/projects: + post: + tags: + - Git Sync + summary: List the GitLab projects a token can sync + description: >- + Lists the projects the supplied GitLab token can push to, so a git + repository resource can be filled in without hand-writing a project + path. The token is used for this call only and is never stored. + Requires workspace admin. + operationId: listGitlabProjects + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + base_url: + type: string + description: The GitLab instance, e.g. https://gitlab.com + token: + type: string + description: A project access token with the api scope, or a group token that reaches the project + search: + type: string + description: Narrow the list to projects matching this text + required: + - base_url + - token + responses: + "200": + description: the projects the token can sync + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/GitlabProject" + + /w/{workspace}/git_sync/credential/origin: + get: + tags: + - Git Sync + summary: Where a repository's credential comes from + description: >- + Whether Windmill holds this repository's access token, and which host it + talks to. `held` means this workspace stores it, `borrowed` means an + ancestor does and it is not this workspace's to replace. Both absent + means the repository authenticates with whatever its URL carries. + Returns no secret. Requires workspace admin. + operationId: getCredentialOrigin + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: path + in: query + required: true + description: >- + Path of the git repository resource, with or without the `$res:` + prefix. A path rather than a URL, because a resource URL may carry a + token and a URL in a query string lands in logs. + schema: + type: string + responses: + "200": + description: where the credential comes from + content: + application/json: + schema: + type: object + properties: + origin: + type: string + enum: + - held + - borrowed + provider: + type: string + enum: + - gitlab + + /w/{workspace}/git_sync/credential: + post: + tags: + - Git Sync + summary: Store the credential for a git repository + description: >- + Stores the access token a git repository authenticates with, so it does + not have to be written into the repository URL or a workspace variable. + The token is write-only: it is served only to a git-sync job that + presents its own job token, and a fork of this workspace reads this + copy instead of holding one of its own. Requires workspace admin. + operationId: setGitCredential + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + repo_url: + type: string + description: >- + The repository the credential is for, and the key it is + stored under. It is served for this repository and no other, + so repointing a resource elsewhere cannot carry the token + along. + token: + type: string + description: The access token, as pasted + required: + - repo_url + - token + responses: + "200": + description: the credential was stored + content: + text/plain: + schema: + type: string + /w/{workspace}/github_app/ghes_installation_callback: post: summary: GHES installation callback @@ -34484,9 +34609,58 @@ components: open_pr_error: type: string description: server-owned, last failure opening a PR for a deploy branch of this repo + credential: + $ref: "#/components/schemas/GitCredentialStatus" required: - git_repo_resource_path + GitCredentialStatus: + type: object + description: server-owned, what the repo's own credential reports about itself + properties: + provider: + type: string + enum: + - gitlab + token_id: + type: integer + format: int64 + expires_at: + type: string + format: date + description: absent for a non-expiring token + scopes: + type: array + items: + type: string + rotatable: + type: boolean + description: whether this workspace renews the credential itself + checked_at: + type: integer + format: int64 + error: + type: string + required: + - provider + - rotatable + - checked_at + + GitlabProject: + type: object + description: a GitLab project a token can sync, as the resource form needs it + properties: + path_with_namespace: + type: string + description: nested group path plus project name, which is also GitLab's project id + http_url_to_repo: + type: string + default_branch: + type: string + required: + - path_with_namespace + - http_url_to_repo + AutoPullMode: type: string enum: diff --git a/backend/windmill-api/src/git_sync_oss.rs b/backend/windmill-api/src/git_sync_oss.rs index 0451d88699..d1653d9aa9 100644 --- a/backend/windmill-api/src/git_sync_oss.rs +++ b/backend/windmill-api/src/git_sync_oss.rs @@ -10,6 +10,11 @@ pub fn workspaced_service() -> Router { Router::new() } +#[cfg(not(feature = "private"))] +pub fn workspaced_git_sync_service() -> Router { + Router::new() +} + #[cfg(not(feature = "private"))] pub fn global_service() -> Router { Router::new() diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index 4ba60428c9..f404d7be9b 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -959,6 +959,15 @@ pub async fn run_server( #[cfg(not(feature = "enterprise"))] Router::new() }) + .nest("/w/{workspace_id}/git_sync", { + #[cfg(feature = "enterprise")] + { + git_sync_oss::workspaced_git_sync_service() + } + + #[cfg(not(feature = "enterprise"))] + Router::new() + }) .nest( "/w/{workspace_id}/resources_u", public_service().layer(cors.clone()), diff --git a/backend/windmill-api/src/workspaces_export.rs b/backend/windmill-api/src/workspaces_export.rs index 1c22981ee0..3a2a608369 100644 --- a/backend/windmill-api/src/workspaces_export.rs +++ b/backend/windmill-api/src/workspaces_export.rs @@ -1586,10 +1586,11 @@ pub(crate) async fn tarball_workspace( .await?; // Use v2 format only if explicitly requested, otherwise use v1 (legacy) for backward compatibility - // Server-owned auto-pull state (the HMAC webhook secret + hook id/error and - // the synced-sha / last-pull status) must never leave the server: keep it out - // of export archives and synced repos, and don't let a re-imported workspace - // inherit another install's hook/sync state. Mirrors the GET-settings redaction. + // Server-owned state (the HMAC webhook secret + hook id/error, the + // synced-sha / last-pull status, and what the credential check observed) + // must never leave the server: keep it out of export archives and synced + // repos, and don't let a re-imported workspace inherit another install's + // hook/sync state. Mirrors the GET-settings redaction. fn redact_git_sync_for_export(git_sync: Option) -> Option { let mut git_sync = git_sync?; if let Some(repos) = git_sync @@ -1610,6 +1611,13 @@ pub(crate) async fn tarball_workspace( auto_pull.remove(field); } } + // What this install observed about its own credential: a token + // id and expiry, and a `checked_at` that moves on its own. + // None of it describes the workspace, and in a git-synced + // `wmill.yaml` it would churn the file for no reason. + if let Some(repo) = repo.as_object_mut() { + repo.remove("credential"); + } } } Some(git_sync) diff --git a/backend/windmill-common/src/git_sync_oss.rs b/backend/windmill-common/src/git_sync_oss.rs index 5f8ba771fb..4c7de4fa7b 100644 --- a/backend/windmill-common/src/git_sync_oss.rs +++ b/backend/windmill-common/src/git_sync_oss.rs @@ -1,11 +1,16 @@ #[cfg(feature = "private")] #[allow(unused)] pub use crate::git_sync_ee::*; -#[cfg(not(feature = "private"))] +#[cfg(not(all(feature = "private", feature = "enterprise")))] use sqlx::{Pool, Postgres}; use url::Url; -#[cfg(not(feature = "private"))] +/// Gated on the pair to match [`with_stored_credential`] below, whose callers +/// reach it through this facade un-gated and so depend on it. Nothing routes +/// here today (the one caller imports the enterprise item directly), so this is +/// for uniformity: the next plain caller would otherwise find no definition +/// under `private` without `enterprise`. +#[cfg(not(all(feature = "private", feature = "enterprise")))] pub async fn get_github_app_token_internal( _db: &Pool, _job_token: &str, @@ -15,6 +20,21 @@ pub async fn get_github_app_token_internal( )); } +/// Server-held git credentials are an enterprise feature, so on this build a +/// repository URL authenticates with whatever it already carries. +/// +/// Gated on the pair rather than on `private` alone: `private` does not imply +/// `enterprise`, and the callers are plain (no `#[cfg]`), so a build with one +/// and not the other would find neither this nor the enterprise definition. +#[cfg(not(all(feature = "private", feature = "enterprise")))] +pub async fn with_stored_credential( + _db: &Pool, + _w_id: &str, + url: String, +) -> crate::error::Result { + Ok(url) +} + lazy_static::lazy_static! { /// Matches a `user:password@` (or `user@`) userinfo component right after the URL scheme. static ref GIT_URL_USERINFO_RE: regex::Regex = diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index b6d10cb7df..443c3b6b63 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -183,7 +183,7 @@ pub enum ObjectType { DatatableMigration, } -pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28931/sync-script-to-git-repo-windmill"; +pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28949/sync-script-to-git-repo-windmill"; /// Hub script that applies a repository's state back into a workspace /// (the repo → Windmill / "pull" direction). Same script the UI runs from @@ -191,7 +191,7 @@ pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28931/sync-script-to-git-repo /// ignores the slug, so the slug is kept free of characters that would be /// percent-encoded into the run URL (a `:` becomes `%3A`, which some hardened /// reverse proxies reject as double-encoding when the client re-encodes it). -pub const GIT_SYNC_PULL_SCRIPT_PATH: &str = "hub/28930/git-sync-init-repository-windmill"; +pub const GIT_SYNC_PULL_SCRIPT_PATH: &str = "hub/28948/git-sync-init-repository-windmill"; /// Prefix used to identify fork workspaces. A workspace whose id starts with this string is a /// fork of another workspace. @@ -345,13 +345,14 @@ pub struct GitRepositorySettings { #[serde(default, skip_serializing_if = "Option::is_none")] pub auto_pull: Option, /// Open a PR when a deploy pushes a `wm_deploy/**` branch of this promotion - /// repo (app-backed only; runs from the deploy callback so it works without + /// repo (needs a credential the server holds — a GitHub App installation or + /// a checked GitLab token; runs from the deploy callback so it works without /// inbound webhooks). Off by default so upgrades don't change behavior. #[serde(default, skip_serializing_if = "is_false")] pub promotion_open_prs: bool, /// Parent-level: open a PR when a fork of this workspace deploys to its - /// `wm-fork/**` branch (app-backed only; the fork's deploy callback reads - /// this from the parent). Off by default. + /// `wm-fork/**` branch (needs a credential the server holds; the fork's + /// deploy callback reads this from the parent). Off by default. #[serde(default, skip_serializing_if = "is_false")] pub fork_open_prs: bool, /// Server-owned: the last failure opening a PR for a deploy branch of this @@ -360,6 +361,10 @@ pub struct GitRepositorySettings { /// successful PR; never accepted from clients. #[serde(default, skip_serializing_if = "Option::is_none")] pub open_pr_error: Option, + /// Server-owned: what the repo's credential says about its own expiry and + /// scopes. Written by the credential check, never accepted from clients. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub credential: Option, } impl GitRepositorySettings { @@ -405,6 +410,44 @@ pub enum AutoPullMode { Polling, } +/// Host whose credential lifecycle Windmill can manage from the repo URL. +#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum GitCredentialProvider { + Gitlab, +} + +/// What the repo's own credential says about itself, refreshed by asking the +/// host. Server-owned: written by the credential check, never accepted from a +/// client. +/// +/// Absent means the check has not run or the repo carries no credential we can +/// introspect (a GitHub App repo mints tokens per call and has nothing to expire). +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct GitCredentialStatus { + pub provider: GitCredentialProvider, + /// Changes on every rotation, so it identifies the current token, not the + /// credential's whole history. + #[serde(skip_serializing_if = "Option::is_none")] + pub token_id: Option, + /// `None` is a non-expiring token, which only self-managed GitLab can issue + /// (and only for a service account). It means no warning and no rotation. + #[serde(skip_serializing_if = "Option::is_none")] + pub expires_at: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub scopes: Vec, + /// Whether *this workspace* renews the credential. That needs a scope which + /// permits it (`api` or `self_rotate`) and a credential this workspace holds: + /// a token carried in the repository URL is the operator's to manage, and one + /// resolved from an ancestor is the ancestor's, so neither is renewed here. + pub rotatable: bool, + /// Unix timestamp (seconds) of the last check. + pub checked_at: i64, + /// Why the last check or rotation failed, cleared by the next success. + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, +} + /// Outcome of the most recent auto-pull attempt, surfaced in the UI. #[derive(Serialize, Deserialize, Debug, Clone)] pub struct AutoPullStatus { diff --git a/backend/windmill-store/src/resources.rs b/backend/windmill-store/src/resources.rs index cf3c73f8da..1ea5545b68 100644 --- a/backend/windmill-store/src/resources.rs +++ b/backend/windmill-store/src/resources.rs @@ -3629,6 +3629,16 @@ async fn get_git_commit_hash( })?; git_resource.url = resolve_azure_devops_url(&db_with_opt_authed, &w_id, &git_resource.url, false).await?; + // A credential is stored under the repository it was issued for, so a + // resource repointed elsewhere finds none. Which credential can be attached + // is bounded by that; who may use it is bounded here, on the same terms as + // the installation credential above. + let plain_url = git_resource.url.clone(); + git_resource.url = + windmill_common::git_sync_oss::with_stored_credential(&db, &w_id, git_resource.url).await?; + if git_resource.url != plain_url { + require_admin(authed.is_admin, &authed.username)?; + } let identities: Vec = query .git_ssh_identity @@ -4307,6 +4317,10 @@ pub async fn get_git_repo_head_for_autopull( } git_resource.url = resolve_azure_devops_url(&git_sync_system_dba(db), w_id, &git_resource.url, true).await?; + // A repo whose credential Windmill holds carries none in its URL, so the + // poller has to attach it here or every probe would be unauthenticated. + git_resource.url = + windmill_common::git_sync_oss::with_stored_credential(db, w_id, git_resource.url).await?; if let Some(branch) = git_resource.branch.as_deref().filter(|s| !s.is_empty()) { let branch = branch.to_string(); @@ -4413,6 +4427,11 @@ pub async fn get_git_repo_fork_heads_for_autopull( )); } git_resource.url = resolve_azure_devops_url(&dba, w_id, &git_resource.url, true).await?; + // Same reason as the head probe above: a repository whose credential Windmill + // holds carries none in its URL, and listing the fork branches is the half of + // polling that would otherwise go out unauthenticated. + git_resource.url = + windmill_common::git_sync_oss::with_stored_credential(db, w_id, git_resource.url).await?; validate_git_url(&git_resource.url).await?; validate_git_ref(base_branch)?; diff --git a/backend/windmill-worker/src/result_processor.rs b/backend/windmill-worker/src/result_processor.rs index 647d569cbe..fd4f08f5fb 100644 --- a/backend/windmill-worker/src/result_processor.rs +++ b/backend/windmill-worker/src/result_processor.rs @@ -816,8 +816,19 @@ pub async fn handle_receive_completed_job( #[cfg(all(feature = "enterprise", feature = "private"))] #[derive(serde::Deserialize)] struct GitSyncCheck { - check_run_id: i64, - repo_url: String, + /// Absent when the repository's host has no check surface (GitLab): the + /// result then reaches the pull request through the managed comment alone. + #[serde(default)] + check_run_id: Option, + /// Only markers written before the repository URL moved out of job args + /// carry one; the resource path on the job is what is used now. + #[serde(default)] + repo_url: Option, + /// Host and path of the repository the check was created on, with no + /// credential in it. The resource path is mutable, so this is what proves + /// the resource still points where the check lives. + #[serde(default)] + repo: Option, #[serde(default)] pr_number: Option, #[serde(default)] @@ -1162,18 +1173,18 @@ async fn maybe_open_git_sync_deploy_pr( }; // Base = the tracked branch (resource branch, else the repo default). Also - // acts as the app-backed gate: PR creation needs the installation token. - let base = match windmill_common::git_sync_ee::get_app_repo_head_for_autopull( + // acts as the gate: PR creation needs a credential the server itself holds. + let base = match windmill_common::git_sync_ee::managed_pr_base_branch( db, workspace_id, &repo_path, ) .await { - Ok(Some((branch, _))) => branch, + Ok(Some(branch)) => branch, Ok(None) => { tracing::warn!( - "git sync PR: repo {repo_path} in {workspace_id} has a PR-on-deploy toggle set but is not GitHub-App-backed; skipping (connect the repo through the GitHub App, or use the open-pr-on-commit workflow)" + "git sync PR: repo {repo_path} in {workspace_id} has a PR-on-deploy toggle set but the server holds no credential for it; skipping (connect the repo through the GitHub App or a GitLab token, or use the open-pr-on-commit workflow)" ); return; } @@ -1373,22 +1384,59 @@ async fn maybe_post_git_sync_check( (None, Some(deploy)) => (true, deploy), (None, None) => return, }; - let Ok(mut check) = serde_json::from_value::(marker) else { + let Ok(check) = serde_json::from_value::(marker) else { return; }; - // Markers carry the literal resource URL (job args are persisted, so a - // `$var:`-resolved URL must not land there); interpolate before calling - // GitHub. - check.repo_url = - match windmill_common::variables::get_variable_or_self(check.repo_url, db, workspace_id) - .await - { - Ok(u) => u, - Err(e) => { - tracing::error!("git sync-check: cannot interpolate repo url: {e:#}"); - return; - } - }; + // Job args are persisted, so the repository URL is not among them: it is + // re-resolved here from the resource path the pull job carries. A marker + // written before that change still has the URL, and is honoured until the + // last such job has drained. + // The resource path is mutable, so it is only trusted when the marker also + // carries the identity to check it against. A marker written before that + // identity existed keeps using the URL it captured at enqueue, which cannot + // have been repointed since. + let repo_url = match ( + check.repo.is_some(), + row.repo_path.as_deref(), + check.repo_url.clone(), + ) { + // The resource path is mutable, so following it is only safe when the + // marker also carries the identity to check the result against. + (true, Some(path), _) => { + windmill_common::git_sync_ee::resolve_repo_url_interpolated(db, workspace_id, path) + .await + } + // A marker written before that identity existed captured the URL itself, + // which cannot have been repointed since. + (_, _, Some(url)) => { + windmill_common::variables::get_variable_or_self(url, db, workspace_id).await + } + // Neither: nothing here can prove which repository this check belongs to, + // and resolving the path anyway is how a preview reaches the wrong one. + // Leaving the check unfinished is the safe failure. + _ => { + tracing::error!( + "git sync-check: the marker carries neither a repository identity nor a url; not acting on it" + ); + return; + } + }; + let repo_url = match repo_url { + Ok(u) => u, + Err(e) => { + tracing::error!("git sync-check: cannot resolve repo url: {e:#}"); + return; + } + }; + // A resource repointed while the diff was running would otherwise close a + // check, or post a preview, on a repository that has nothing to do with it. + if check.repo.is_some() && windmill_common::git_sync_ee::repo_identity(&repo_url) != check.repo + { + tracing::warn!( + "git sync-check: the repository moved since the check was created; leaving it alone" + ); + return; + } // "In sync" on a PR that visibly changes files reads as a bug when those // files are outside the repo's sync filters — say what the scope is. let scope_note = if !is_deploy && success { @@ -1451,7 +1499,7 @@ async fn maybe_post_git_sync_check( ( "failure", "Merge conflicts with the base branch".to_string(), - "This PR cannot be merged cleanly, so there is no deploy diff to compute. Resolve the conflicts and push again to re-run this check." + "This branch cannot be merged cleanly, so there is no deploy diff to compute. Resolve the conflicts and push again to re-run this check." .to_string(), ) } else if pr_check_error.as_deref() == Some("PR_HEAD_REF_UNAVAILABLE") { @@ -1461,7 +1509,7 @@ async fn maybe_post_git_sync_check( ( "neutral", "Could not compute the deploy diff".to_string(), - "Windmill could not fetch this PR's head or enough history from GitHub to compute its merge with the base. Push again to re-run this check." + "Windmill could not fetch this branch's head, or enough history, to compute its merge with the base. Push again to re-run this check." .to_string(), ) } else if pr_check_error.is_some() { @@ -1484,20 +1532,20 @@ async fn maybe_post_git_sync_check( "success", "In sync".to_string(), format!( - "Merging this PR would make no changes to the workspace.{}", + "Merging this branch would make no changes to the workspace.{}", scope_note.as_deref().unwrap_or_default() ), ), Some((changes, settings_changed)) => { let mut lines = vec![format!( - "Merging this PR would apply {} change(s) to the workspace:\n", + "Merging this branch would apply {} change(s) to the workspace:\n", changes.len() )]; lines.extend(format_change_list(&changes)); if settings_changed { lines.push(match check.wmill_yaml_changed { - Some(true) => "\nThis PR changes wmill.yaml: pulling also applies the updated workspace settings.".to_string(), - Some(false) => "\nIndependent of this PR, the workspace's git-sync settings differ from the repo's wmill.yaml and a pull updates them to match.".to_string(), + Some(true) => "\nThis branch changes wmill.yaml: pulling also applies the updated workspace settings.".to_string(), + Some(false) => "\nIndependent of this branch, the workspace's git-sync settings differ from the repo's wmill.yaml and a pull updates them to match.".to_string(), None => "\nA pull also updates the workspace's git-sync settings to match the repo's wmill.yaml.".to_string(), }); } @@ -1520,19 +1568,21 @@ async fn maybe_post_git_sync_check( Some(url) => format!("{summary}\n\n[See the job in Windmill]({url})"), None => summary.clone(), }; - if let Err(e) = windmill_common::git_sync_ee::update_check_run( - db, - workspace_id, - &check.repo_url, - check.check_run_id, - conclusion, - &title, - &check_summary, - job_url.as_deref(), - ) - .await - { - tracing::error!("git sync-check: failed to update check run: {e:#}"); + if let Some(check_run_id) = check.check_run_id { + if let Err(e) = windmill_common::git_sync_ee::update_check_run( + db, + workspace_id, + &repo_url, + check_run_id, + conclusion, + &title, + &check_summary, + job_url.as_deref(), + ) + .await + { + tracing::error!("git sync-check: failed to update check run: {e:#}"); + } } // Phase 4 also maintains ONE managed comment on the PR (Cloudflare @@ -1556,7 +1606,7 @@ async fn maybe_post_git_sync_check( if let Err(e) = windmill_common::git_sync_ee::upsert_pr_comment( db, workspace_id, - &check.repo_url, + &repo_url, pr_number, marker, &body, diff --git a/docs/git-sync-gitlab-setup.md b/docs/git-sync-gitlab-setup.md new file mode 100644 index 0000000000..6e736dee37 --- /dev/null +++ b/docs/git-sync-gitlab-setup.md @@ -0,0 +1,170 @@ +# Git sync with GitLab + +GitLab has no equivalent of a GitHub App, so there is nothing to install and no +consent screen. What Windmill needs instead is one credential you create in +GitLab and paste once. With it, a GitLab repository gets the same managed +features an app-backed GitHub repository has: instant pull over a webhook, merge +requests opened on deploy, and a diff preview posted onto the merge request. + +## The credential + +Create a **project access token** on the project you are syncing (Settings → +Access tokens). It is a bot identity that outlives the person who created it, +which is what you want for a credential the instance uses unattended, and it +reaches exactly the one project. + +| | | +| --- | --- | +| Scope | `api` | +| Role | Developer to push deploy branches; **Maintainer** to also manage the webhook and open merge requests | +| Expiry | Required. A group service account PAT can be non-expiring on self-managed (see below); an access token cannot | + +**Use a separate token per repository.** A group access token works too and +reaches every project in the group, which is convenient for a lot of +repositories — but Windmill stores the credential per repository, and renewal +rewrites the repository it renewed for. Any other repository holding that same +token keeps the revoked one and stops syncing until you paste a new token there. +Each stranded repository says so on its card, so it is visible rather than +silent, but a token per repository avoids it entirely. + +`api` is a superset: it authorizes Git over HTTPS as well, so no separate +`write_repository` is needed to clone and push, and it is also what makes the +token renewable. A `write_repository`-only token can still push, but Windmill +cannot inspect or renew it and reports that in the workspace's git sync settings. + +### The identity Windmill acts as + +GitLab issues an access token to a bot user it creates for it — `project__bot_…` +for a project token, `group__bot_…` for a group one — and the bot's display +name is **the name you gave the token**. That name is the byline on everything +Windmill does: the author of deploy commits, of the merge requests it opens, and +of the preview notes it writes. Name it for what it is, `windmill-sync` or +similar, rather than something only you will recognise. + +Each token you create adds another bot member to the project or group. Renewal +does not — it keeps the same bot — so a repository accumulates one bot, not one +per year. + +Renewal goes through GitLab's own self-rotation endpoint. Both kinds of access +token are held as their bot user's personal access token, so the token rotates +itself and Windmill never needs a credential with rights over the project or +group. + +## Connecting a repository + +In the resource form for a `git_repository` resource, use the **GitLab** button: +paste the instance URL and the token, pick a project from the list, and Windmill +keeps the token for you. The resource itself gets the plain remote URL +(`"url": "https://gitlab.com/group/project.git"`), with no credential in it. + +The token is stored encrypted on the workspace, keyed by the repository it was +issued for rather than by the resource naming it, the same way a GitHub App +installation is held against the account it covers. Nothing reads it back out +over the API: the server attaches it when it talks to GitLab, and a sync job +receives it only against its own job token. Repointing a resource's `url` asks +for a different repository's token and finds none, so the edit carries nothing +with it; a repository that genuinely moved needs its token entered again. + +Because the repository is the key, the token is stored the moment you pick the +project, before the resource is saved. Renaming the resource later keeps it, and +cancelling the edit leaves a stored token that nothing uses until some resource +points at that repository again. + +Forks of the workspace read this one copy rather than getting their own, so +renewal reaches all of them at once and the token is not duplicated into every +descendant workspace. + +Treat workspace admin as equivalent to holding the token. An admin of the +workspace, or of any fork below it, can point a repository at a sync script they +wrote and have that job request the credential, exactly as they can for a GitHub +App installation token. Storing it this way keeps it out of the variables API and +out of every fork's own storage; it is not a boundary against the admins of those +workspaces. + +A URL with the token written into it, in the resource or in a secret variable +the resource points at (`"url": "$var:..."`), is a plain git remote: it syncs on +deploy and by polling, and nothing else. Windmill does not know the token is +there, so it registers no webhook, opens no merge request, and neither reports +nor renews its expiry. Use the **GitLab** button to hand the token to Windmill +if you want any of that. + +## Expiry and renewal + +Windmill reads `expires_at` from the token it holds and shows it on the +repository in the workspace's git sync settings. Within three weeks of expiry it +rotates the token through GitLab's own +`POST /personal_access_tokens/self/rotate`, stores the replacement, and verifies +it. Only the token can rotate itself, so one without `api` (or `self_rotate`) is +a permanent warning rather than something Windmill can fix. + +Only the workspace that holds a credential renews it, so one renewal serves the +whole fork chain instead of each fork racing to renew the same token. A fork +reads the parent's without holding one, so it never renews; the parent does, and +every fork sees the replacement at once. + +Rotation is deliberately never retried. GitLab revokes the old token the instant +it issues the replacement, and presenting an already-rotated token to `/rotate` +again is treated as reuse: it revokes **the whole token family, including the +live replacement**. So a rotation that succeeded at GitLab but failed to persist +is surfaced as an error to act on, not retried. + +Non-expiring tokens are possible only for a **group service account PAT** on +self-managed, with `require_personal_access_token_expiry` turned off in the +instance's application settings. A group access token is always rejected without +an `expires_at`. + +## What each managed feature needs + +| Feature | Needs | +| --- | --- | +| Instant pull | A project hook Windmill creates, so Maintainer; and a Windmill base URL GitLab can reach | +| Merge requests on deploy | Developer, plus the `api` scope | +| Diff preview on a merge request | The project hook, plus permission to post merge request notes | + +Instant pull falls back to checking the tracked branch about every minute when +the hook cannot be created or delivered, so nothing silently stops syncing. + +## Self-managed differences + +**Webhooks to a private network are blocked by default.** GitLab refuses to +create a hook pointing at a private or local address until an administrator +enables *Allow requests to the local network from webhooks and integrations* +(Admin → Settings → Network → Outbound requests, +`allow_local_requests_from_web_hooks_and_services`). A Windmill instance on the +same private network as GitLab needs this; without it, hook creation fails with a +"blocked" error and the repository keeps polling. + +**A relative-URL install is not supported.** GitLab can be served under a path +prefix (`https://example.com/gitlab`), and that prefix cannot be told apart from +a group of the same name: `example.com/a/b/c.git` is either group `a/b` project +`c`, or prefix `a` with group `b` project `c`. Windmill reads it as the nested +group, so on a relative-URL install it derives the wrong API base and the managed +features stay unavailable. Such a repository still syncs through its token URL, +which needs no API base. + +Everything else is identical: Windmill talks to `/api/v4` and needs +no inbound access of its own beyond the hook deliveries. + +## The deploy preview is a note, not a pipeline status + +On GitHub the preview is a check run: its own object, advisory unless the +repository makes it required. GitLab has no equivalent. Its only comparable +primitive is a commit status, and posting one has side effects Windmill will not +impose on a project: + +- GitLab files the status **as a job inside whatever pipeline already covers that + commit**, so a failed Windmill status fails the project's own pipeline, and its + reviewers see their test suite as failed. +- `allow_failure` is ignored on the commit-status endpoint, so the status cannot + be made advisory. +- On a commit with no pipeline it creates an `external` pipeline instead, which + then gates merging under *Pipelines must succeed*, including while it is still + running. + +So on GitLab the preview lives entirely in a **merge request note** that Windmill +keeps up to date: it carries the workspace, the status line, the commit, a link +to the job, and the full list of changes merging would deploy. A note cannot +block a merge or change what the project's own CI reports. + +The note is upserted rather than appended, so a merge request accumulates one +Windmill comment however many times it is pushed to. diff --git a/frontend/src/lib/components/ApiConnectForm.svelte b/frontend/src/lib/components/ApiConnectForm.svelte index e4ee3ca7b9..dedeee90e2 100644 --- a/frontend/src/lib/components/ApiConnectForm.svelte +++ b/frontend/src/lib/components/ApiConnectForm.svelte @@ -12,6 +12,7 @@ import { Loader2 } from 'lucide-svelte' import { untrack } from 'svelte' import GitHubAppIntegration from './GitHubAppIntegration.svelte' + import GitLabIntegration from './GitLabIntegration.svelte' import BedrockCredentialsCheck from './BedrockCredentialsCheck.svelte' import { isCloudHosted } from '$lib/cloud' import ResourceGen from './copilot/ResourceGen.svelte' @@ -28,6 +29,13 @@ isValid?: boolean linkedSecretCandidates?: string[] | undefined description?: string | undefined + /** Workspace the resource is being saved into, which is not always the one + * being navigated. The GitLab picker has to store the credential where the + * resource will look for it. */ + workspace?: string + /** Fired once the GitLab picker has stored the picked project's token, so a + * form that would otherwise file the URL as a secret knows it holds none. */ + onCredentialStored?: () => void onSynced?: () => void } @@ -39,6 +47,8 @@ isValid = $bindable(true), linkedSecretCandidates = undefined, description = $bindable(undefined), + workspace = undefined, + onCredentialStored, onSynced = undefined }: Props = $props() @@ -249,6 +259,19 @@ }} onDescriptionUpdate={(newDescription) => (description = newDescription)} /> + + { + args = newArgs + rawCode = JSON.stringify(args, null, 2) + rawCodeEditor?.setCode(rawCode) + }} + />
{#if resourceType?.includes('bedrock') && !isCloudHosted()} diff --git a/frontend/src/lib/components/AppConnectInner.svelte b/frontend/src/lib/components/AppConnectInner.svelte index 7f2b646f6b..b0655590c2 100644 --- a/frontend/src/lib/components/AppConnectInner.svelte +++ b/frontend/src/lib/components/AppConnectInner.svelte @@ -1373,6 +1373,15 @@ {linkedSecretCandidates} {resourceType} {resourceTypeInfo} + workspace={effectiveWorkspace} + onCredentialStored={() => { + // `forceSecretValue` files a git_repository's `url` in a secret + // variable, for the URLs that carry a token in them. The picker's + // does not: the token is stored separately, so that variable would + // hold nothing secret and add a second place to keep in step with + // the resource. + linkedSecrets = linkedSecrets.filter((f) => f !== 'url') + }} bind:args bind:isValid onSynced={getResourceTypeInfo} diff --git a/frontend/src/lib/components/GitHubAppIntegration.svelte b/frontend/src/lib/components/GitHubAppIntegration.svelte index 11844d7d9c..89a80de6b0 100644 --- a/frontend/src/lib/components/GitHubAppIntegration.svelte +++ b/frontend/src/lib/components/GitHubAppIntegration.svelte @@ -180,7 +180,7 @@ {#if !githubState.loadingGithubInstallations} + {/snippet} + {#snippet content({ close })} +
+
+
+
GitLab instance
+ +
+
+
Project access token
+
+ Create it on the project you are syncing, with the api scope and the + Maintainer role. +
+ +
+ Windmill stores it and renews it before it expires. Use one token per project: a group + token covers the group, but renewal replaces it for one project at a time. +
+
+
+ +
+ {#if listError} + {listError} + {/if} + {#if staleListing} +
+ The token changed. List the projects again to choose one it can reach. +
+ {/if} + {#if projects.length > 0 && !staleListing} +
+
Project
+
+ + + {/snippet} + + {#snippet empty()} +

+ {#if loadFailed} + Could not reach the hub. You can still browse its projects in a new tab. + {:else} + This hub has no projects yet. + {/if} +

+ {/snippet} + + + diff --git a/frontend/src/lib/components/home/ImportProjectModal.svelte b/frontend/src/lib/components/home/ImportProjectModal.svelte new file mode 100644 index 0000000000..d9ac59d3b3 --- /dev/null +++ b/frontend/src/lib/components/home/ImportProjectModal.svelte @@ -0,0 +1,363 @@ + + +{#snippet importPage()} +
+ {#if project} + + + {/if} + + (folder = f)} + onFinish={() => (setup.needed ? (onSetupStep = true) : finish('none'))} + onBack={onClose} + onExecution={(e) => (execution = e)} + resume={execution} + /> +
+{/snippet} + +{#snippet setupPlaceholder()} + +
+ +
+{/snippet} + +{#snippet setupPage()} +
+ finish('skipped', outstanding)} + onFinish={(checked) => finish(checked ? 'filled' : 'unchecked')} + onBack={execution ? () => (onSetupStep = false) : undefined} + /> +
+{/snippet} + + + + {#if slug} + + (onSetupStep = s === 2)} + /> + + { + if (key === IMPORT_PAGE && execution) onSetupStep = false + else if (key === SETUP_PAGE && setup.needed) onSetupStep = true + }} + pages={[ + { key: IMPORT_PAGE, content: importPage }, + { key: SETUP_PAGE, content: setupPage, placeholder: setupPlaceholder } + ]} + /> + {/if} + diff --git a/frontend/src/lib/components/home/ItemsList.svelte b/frontend/src/lib/components/home/ItemsList.svelte index 84432112a3..95f95bf6ce 100644 --- a/frontend/src/lib/components/home/ItemsList.svelte +++ b/frontend/src/lib/components/home/ItemsList.svelte @@ -16,7 +16,7 @@ } from '$lib/gen' import { resource } from 'runed' import { getDraftItems } from '$lib/workspaceDrafts.svelte' - import { userStore, workspaceStore } from '$lib/stores' + import { disableHubStore, userStore, workspaceStore } from '$lib/stores' import type uFuzzy from '@leeoniya/ufuzzy' import { ArrowDownUp, @@ -40,6 +40,10 @@ type FilterSchemaRec } from '$lib/components/FilterSearchbar.svelte' import NoItemFound from './NoItemFound.svelte' + import WorkspaceEmptyState from './WorkspaceEmptyState.svelte' + import HubProjectPickerModal from './HubProjectPickerModal.svelte' + import ImportProjectModal from './ImportProjectModal.svelte' + import type { HubProjectPick } from '$lib/hubProject' import ListFilters from './ListFilters.svelte' import ToggleButtonGroup from '../common/toggleButton-v2/ToggleButtonGroup.svelte' import ToggleButton from '../common/toggleButton-v2/ToggleButton.svelte' @@ -696,6 +700,10 @@ // runnables an owner holds. A scope change (sort/archive/kind/…) doesn't go // through here: the counts resource keys on those itself. async function reloadItemsAndCounts(): Promise { + // The answer can change with the rows: archiving the last item leaves the listing empty + // with something archived behind it, and a cached "nothing archived" would then call + // the workspace empty and hide the way to it until a page load. + archivedProbe = undefined // A mutated row can be gone, or sit at a new path, afterwards: snapshot what // was on screen so the selection can drop what this reload removes instead of // keeping a dead path. `tick` lets the reloaded rows re-register first. @@ -986,6 +994,105 @@ treeLazyMode && ownerCountsRes.current == undefined && ownerCountsRes.loading ) + // An import just landed, so the rows about to replace the empty state are all new: they + // fade in one after another rather than appearing as a finished list. Cleared on a timer + // because nothing else marks the end — the reload resolves before the rows animate. + let justImported = $state(false) + let justImportedTimer: ReturnType | undefined + function onImported() { + reloadItemsAndCounts() + justImported = true + clearTimeout(justImportedTimer) + justImportedTimer = setTimeout(() => (justImported = false), 2500) + } + + // The hub import, owned here rather than by either entry point: the empty state's link and + // the create menu's Import section open the same dialog, and mounting one per entry point + // would put two of them on the page at once while the workspace is still empty. + let hubPick = $state(undefined) + let hubPickerOpen = $state(false) + + /** + * Whether a workspace the default listing found empty is empty at all, or just has nothing + * unarchived — two different states that want two different things said about them. Asked + * only in that case, and once per workspace: one request for one row, never on a workspace + * with something in it. `hasArchived` is undefined when the request failed — see the catch + * for what that leaves standing. + */ + let archivedProbe = $state<{ workspace: string; hasArchived: boolean | undefined } | undefined>( + undefined + ) + $effect(() => { + const ws = $workspaceStore + if (!ws || !workspaceEmpty || archivedProbe?.workspace === ws) return + untrack(() => void probeArchived(ws)) + }) + async function probeArchived(workspace: string) { + try { + // `includeWithoutMain` to match the listing: the backend drops `auto_kind = 'lib'` + // without it, so a workspace holding only archived library scripts would answer + // "nothing archived". Always true here — hiding library scripts puts a filter in + // `activeFilters`, which `workspaceEmpty` requires to be empty. + const res = await ScriptService.listRunnables({ + workspace, + showArchived: true, + includeWithoutMain: true, + perPage: 1 + }) + archivedProbe = { workspace, hasArchived: (res.items?.length ?? 0) > 0 } + } catch (error) { + // Undefined, not false: false would say the workspace is empty and — since the + // toolbar is inert on the strength of the placeholder carrying the way to archived + // items — leave no way to them at all. Unknown keeps the ordinary caption, which + // promises nothing, and leaves the searchbar live as the fallback it used to be. + console.error('Could not check for archived items:', error) + archivedProbe = { workspace, hasArchived: undefined } + } + } + let emptyStateAnswered = $derived(archivedProbe?.workspace === $workspaceStore) + /** + * The probe could not tell. The toolbar stays usable in that case: `inert` is only right + * while the placeholder is the way to archived items, and here it cannot be. + */ + let archivedUnknown = $derived(emptyStateAnswered && archivedProbe?.hasArchived === undefined) + /** + * Whether this user may be offered the create actions. The empty state's template import + * and create menu do no permission check of their own, so an operator — or a workspace + * whose direct-deploy protection cleared `showEditButtons` — must not be shown them. + * Reading archived items is not a write, so it is not gated on this. + */ + let canCreateHere = $derived(!$userStore?.operator && showEditButtons) + + // The workspace itself holds nothing — no filter is narrowing the list away. It stays + // false until the first load resolves: a skeleton already means "loading", and the + // empty state must not be mistaken for one. The controls it dims stay mounted, so + // nothing moves when the first item lands. + let workspaceEmpty = $derived( + !loading && + !treeCountsPending && + !contentActive && + activeFilters.length === 0 && + filteredItems != undefined && + filteredItems.length === 0 && + visiblePipelineFolders.size === 0 && + !hasMoreServer + ) + /** + * Whether the placeholder below takes the toolbar's job over — it renders under the same + * conditions. Standing the toolbar down depends on something else offering a way onwards: + * where the placeholder holds back, as it does for an operator in a workspace that is + * simply empty, these controls are all there is and stay live. + */ + let placeholderTakesOver = $derived( + workspaceEmpty && emptyStateAnswered && (archivedProbe?.hasArchived === true || canCreateHere) + ) + /** + * The toolbar is dimmed either way; `inert` also takes it off the pointer, which is only + * right while the placeholder carries the way to archived items. A probe that could not + * tell leaves it live as the fallback. + */ + let toolbarInert = $derived(placeholderTakesOver && !archivedUnknown) + // Owners the counts found the user has something in, split by kind. They cover // what the folder/username lists miss: an item shared individually out of a // folder or user space the user is otherwise not a member of. @@ -1651,7 +1758,12 @@ }} > {#if !contentActive} -
+ +
{ @@ -1692,9 +1804,10 @@
{/if} - {#if !loading && !contentActive} + {#if !loading && !contentActive && !workspaceEmpty} + view, expand/collapse (tree only), sort. Nothing to select, group or order on + an empty workspace, so the whole row goes. -->
{#if homeSelection.available && !homeSelection.active}
{/if} - {#if filteredItems?.length == 0} + {#if filteredItems?.length == 0 && !workspaceEmpty}
{/if}
@@ -1820,7 +1941,28 @@ - + {#if workspaceEmpty} + + {#if emptyStateAnswered} + {#if archivedProbe?.hasArchived || canCreateHere} + + (hubPick = project)} + onShowArchived={() => (filterValues.val = { ...filterValues.val, archived: true })} + /> + {:else} + + {/if} + {/if} + {:else} + + {/if} {#if hasMoreServer && !searching} @@ -1863,7 +2005,7 @@ /> {/key} {:else} -
+
{#if filter === ''} {#each [...visiblePipelineFolders].sort() as folder (folder)} {/if} + + (hubPickerOpen = false)} + onPick={(project) => { + hubPickerOpen = false + hubPick = project + }} +/> + (hubPick = undefined)} {onImported} /> + + diff --git a/frontend/src/lib/components/home/TutorialBanner.svelte b/frontend/src/lib/components/home/TutorialBanner.svelte deleted file mode 100644 index ef7b30c27e..0000000000 --- a/frontend/src/lib/components/home/TutorialBanner.svelte +++ /dev/null @@ -1,178 +0,0 @@ - - -{#if !isDismissed} - -
- - {#if hasCompletedAny} - New tutorial available! - {:else} - First time? - {/if} - - - -
-{/if} diff --git a/frontend/src/lib/components/home/TutorialButton.svelte b/frontend/src/lib/components/home/TutorialButton.svelte deleted file mode 100644 index 6a31ae9569..0000000000 --- a/frontend/src/lib/components/home/TutorialButton.svelte +++ /dev/null @@ -1,124 +0,0 @@ - - - - diff --git a/frontend/src/lib/components/home/WorkspaceEmptyState.svelte b/frontend/src/lib/components/home/WorkspaceEmptyState.svelte new file mode 100644 index 0000000000..fc36cc334b --- /dev/null +++ b/frontend/src/lib/components/home/WorkspaceEmptyState.svelte @@ -0,0 +1,146 @@ + + +
+ {#each rowOpacities as opacity, i (i)} + + {/each} + + +
+ {#if archivedOnly} + + + Everything in this workspace is archived. + . + + {:else} + Your scripts, flows and apps will show up here. + {/if} + {#if canCreate} + + {#if !$disableHubStore} + + + e.detail && logFeatureUsage('home', 'template_picker_open', { key: 'empty_state' })} + > + {#snippet trigger()}Start from a template{/snippet} + {#snippet content({ close })} + { + close() + onPick(project) + }} + /> + {/snippet} + + or + {/if} + + {#snippet trigger()} + + . + {/snippet} + + {/if} +
+
diff --git a/frontend/src/lib/components/sidebar/OperatorMenu.svelte b/frontend/src/lib/components/sidebar/OperatorMenu.svelte index 0914068607..fa59d730df 100644 --- a/frontend/src/lib/components/sidebar/OperatorMenu.svelte +++ b/frontend/src/lib/components/sidebar/OperatorMenu.svelte @@ -12,10 +12,11 @@ Building, Calendar, ServerCog, - GraduationCap, - Table2 + Table2, + GraduationCap } from 'lucide-svelte' import { base } from '$lib/base' + import { TOUR_PARAM, TOUR_PARAM_VALUE } from '$lib/components/tutorials/operatorTour' import MultiplayerMenu from './MultiplayerMenu.svelte' import { Plus } from 'lucide-svelte' @@ -25,9 +26,7 @@ superadmin, usedTriggerKinds, userWorkspaces, - workspaceStore, - tutorialsToDo, - skippedAll + workspaceStore } from '$lib/stores' import { twMerge } from 'tailwind-merge' import { USER_SETTINGS_HASH } from './settings' @@ -56,22 +55,10 @@ [ { label: 'Home', id: 'home', href: `${base}/`, icon: Home }, { label: 'Runs', id: 'runs', href: `${base}/runs`, icon: Play }, - { label: 'Schedules', id: 'schedules', href: `${base}/schedules`, icon: Calendar }, - // Add Tutorials to main menu only if not all completed and not skipped - ...($tutorialsToDo.length > 0 && !$skippedAll - ? [ - { - label: 'Tutorials', - id: 'tutorials', - href: `${base}/tutorials`, - icon: GraduationCap - } - ] - : []) + { label: 'Schedules', id: 'schedules', href: `${base}/schedules`, icon: Calendar } ].filter( (link) => link.id === 'home' || - link.id === 'tutorials' || ($userWorkspaces && $workspaceStore && $userWorkspaces.find((_) => _.id === $workspaceStore)?.operator_settings?.[link.id] === @@ -243,6 +230,21 @@ Account settings + + + + Take the tour +
diff --git a/frontend/src/lib/components/sidebar/SettingsMenu.svelte b/frontend/src/lib/components/sidebar/SettingsMenu.svelte index fbc07a775f..b7a3a363bb 100644 --- a/frontend/src/lib/components/sidebar/SettingsMenu.svelte +++ b/frontend/src/lib/components/sidebar/SettingsMenu.svelte @@ -12,7 +12,6 @@ Building, Moon, Sun, - GraduationCap, BookOpen, Github, Newspaper, @@ -120,7 +119,6 @@ } const helpItems: Item[] = [ - { displayName: 'Tutorials', icon: GraduationCap, href: `${base}/tutorials` }, { displayName: 'Docs', icon: BookOpen, diff --git a/frontend/src/lib/components/sidebar/SidebarContent.svelte b/frontend/src/lib/components/sidebar/SidebarContent.svelte index fb1bb9ee17..7fa161dd2d 100644 --- a/frontend/src/lib/components/sidebar/SidebarContent.svelte +++ b/frontend/src/lib/components/sidebar/SidebarContent.svelte @@ -8,12 +8,9 @@ workspaceStore, isCriticalAlertsUIOpen, enterpriseLicense, - devopsRole, - tutorialsToDo, - skippedAll + devopsRole } from '$lib/stores' import { isForkOwner } from '$lib/utils/workspaceHierarchy' - import { syncTutorialsTodos } from '$lib/tutorialUtils' import { SIDEBAR_SHOW_SCHEDULES } from '$lib/consts' import { BookOpen, @@ -26,7 +23,6 @@ FolderCog, FolderOpen, Github, - GraduationCap, HelpCircle, Home, LogOut, @@ -51,7 +47,6 @@ import DiscordIcon from '../icons/brands/Discord.svelte' import ConfirmationModal from '../common/confirmationModal/ConfirmationModal.svelte' import { twMerge } from 'tailwind-merge' - import { onMount } from 'svelte' import { base } from '$lib/base' import { page } from '$app/state' import SideBarNotification from './SideBarNotification.svelte' @@ -116,11 +111,6 @@ 'boolean' ) - onMount(async () => { - // Sync tutorial progress on mount - await syncTutorialsTodos() - }) - function openChangelogs() { markChangelogsOpened() hasNewChangelogs = false @@ -131,14 +121,6 @@ label: 'Help', icon: HelpCircle, subItems: [ - { - label: 'Tutorials', - href: `${base}/tutorials`, - icon: GraduationCap, - aiId: 'sidebar-menu-link-tutorials', - aiDescription: 'Button to navigate to tutorials', - external: false - }, { label: 'Docs', href: 'https://www.windmill.dev/docs/intro/', @@ -269,19 +251,7 @@ disabled: $userStore?.operator, aiId: 'sidebar-menu-link-groups', aiDescription: 'Button to navigate to groups' - }, - // Add Tutorials to main menu only if not all completed and not skipped - ...($tutorialsToDo.length > 0 && !$skippedAll - ? [ - { - label: 'Tutorials', - href: `${base}/tutorials`, - icon: GraduationCap, - aiId: 'sidebar-menu-link-tutorials-main', - aiDescription: 'Button to navigate to tutorials' - } - ] - : []) + } ].filter((l) => !excludeMainLabels.includes(l.label)) ) let defaultExtraTriggerLinks = $derived([ diff --git a/frontend/src/lib/components/tutorials/FlowBuilderLiveTutorial.svelte b/frontend/src/lib/components/tutorials/FlowBuilderLiveTutorial.svelte deleted file mode 100644 index 343deb917c..0000000000 --- a/frontend/src/lib/components/tutorials/FlowBuilderLiveTutorial.svelte +++ /dev/null @@ -1,754 +0,0 @@ - - - { - const steps: DriveStep[] = [ - { - popover: { - title: 'Build your first flow', - description: - "Let's create a temperature converter that validates input and converts Celsius to Fahrenheit.", - onNextClick: async () => { - const emptyFlow: Flow = { - summary: '', - description: '', - value: { modules: [] }, - schema: flowJson.schema, - path: '', - edited_at: '', - edited_by: '', - archived: false, - extra_perms: {} - } - await initFlow(emptyFlow, flowStore as StateStore, flowStateStore) - - driver.moveNext() - } - } - }, - { - element: '#flow-editor-virtual-Input', - onHighlighted: async () => { - step2Complete = false - - await wait(DELAY_MEDIUM) - triggerPointerDown('#flow-editor-virtual-Input') - await wait(DELAY_SHORT) - selectionManager.selectId('Input') - await wait(200) - - const overlay = getDriverOverlay() - if (overlay) { - overlay.style.width = '50%' - overlay.style.right = 'auto' - overlay.style.left = '0' - } - - const celsiusInput = document.querySelector( - 'input[type="number"][placeholder=""]' - ) as HTMLInputElement - if (celsiusInput) { - celsiusInput.value = '' - celsiusInput.dispatchEvent(new Event('input', { bubbles: true })) - await wait(DELAY_MEDIUM) - - celsiusInput.value = '2' - celsiusInput.dispatchEvent(new Event('input', { bubbles: true })) - await wait(400) - - celsiusInput.value = '25' - celsiusInput.dispatchEvent(new Event('input', { bubbles: true })) - - step2Complete = true - } - }, - popover: { - title: 'Set the input', - description: 'Every flow starts with input. Here we define a temperature in Celsius.', - side: 'bottom', - align: 'start', - onNextClick: () => { - if (!step2Complete) { - sendUserToast('Please wait for the input to be filled...', false, [], undefined, 3000) - return - } - driver.moveNext() - } - } - }, - { - element: '#flow-editor-add-step-0', - onHighlighted: async () => { - step3Complete = false - - // Animate cursor to the add step button - const button = document.querySelector('#flow-editor-add-step-0') as HTMLElement - if (button) { - const fakeCursor1 = await createFakeCursorWithStart(null, button, 1.5) - await wait(DELAY_SHORT) - button.click() - fakeCursor1.remove() - } - - const overlay = getDriverOverlay() - if (overlay) { - overlay.style.display = 'none' - } - - await wait(DELAY_LONG) - - const spans = Array.from(document.querySelectorAll('span')) - const bunSpan = spans.find((span) => - span.textContent?.includes('TypeScript (Bun)') - ) as HTMLElement - - if (bunSpan) { - // Animate cursor from add step button to TypeScript (Bun) span - const fakeCursor2 = await createFakeCursorWithStart(button, bunSpan, 1.5) - await wait(DELAY_MEDIUM) - fakeCursor2.remove() - - // Automatically trigger next step after cursor animation - await wait(DELAY_SHORT) - - // Add module with empty summary and empty content - const moduleData = flowJson.value.modules[0] - const module: FlowModule = { - id: moduleData.id, - summary: '', // Start with empty summary - value: moduleData.value - } - // Clear content after module creation if it's a rawscript - if ('content' in module.value) { - module.value = { ...module.value, content: '' } as typeof module.value - } - - await addModuleToFlow(module) - - await wait(700) - - // Restore overlay - const overlay = getDriverOverlay() - if (overlay) { - overlay.style.display = '' - } - - step3Complete = true - driver.moveNext() - } - }, - popover: { - title: 'Choose TypeScript', - description: 'Pick TypeScript (Bun) to write our validation script.', - side: 'top', - onNextClick: () => { - if (!step3Complete) { - sendUserToast( - 'Please wait for the script to be created...', - false, - [], - undefined, - 3000 - ) - return - } - driver.moveNext() - }, - onPrevClick: () => { - sendUserToast('Previous is not available for this step', true, [], undefined, 3000) - } - } - }, - { - element: '#a', - onHighlighted: async () => { - // Reset the flag when step starts - step4Complete = false - - selectionManager.selectId('a') - await wait(DELAY_LONG) - - const overlay = getDriverOverlay() - if (overlay) { - overlay.style.width = '50%' - overlay.style.right = 'auto' - overlay.style.left = '0' - } - - // First, type the summary - await wait(DELAY_MEDIUM) - const summaryInput = document.querySelector( - 'input[placeholder="Summary"]' - ) as HTMLInputElement - if (summaryInput) { - const summaryText = 'Validate temperature input' - await typeText(summaryInput, summaryText) - updateModuleSummary('a', summaryText) - await wait(DELAY_LONG) - } - - // Then, type the code - let editorState = get(currentEditor) - let attempts = 0 - while (attempts < 20) { - if (editorState && editorState.type === 'script' && editorState.stepId === 'a') { - break - } - await wait(100) - editorState = get(currentEditor) - attempts++ - } - - if (editorState && editorState.type === 'script') { - const editor = editorState.editor - const moduleA = flowJson.value.modules.find((m) => m.id === 'a') - const codeToType = - moduleA?.value && 'content' in moduleA.value ? moduleA.value.content : '' - - if (codeToType) { - editor.setCode('', true) - await wait(200) - - let currentText = '' - for (let i = 0; i < codeToType.length; i++) { - const char = codeToType[i] - currentText += char - editor.setCode(currentText, true) - const delay = char === '\n' ? DELAY_CODE_NEWLINE : DELAY_CODE_CHAR - await wait(delay) - } - - // Update the flow store with the typed code - const moduleIndex = flowStore.val.value.modules.findIndex((m) => m.id === 'a') - if ( - moduleIndex !== -1 && - 'content' in flowStore.val.value.modules[moduleIndex].value - ) { - flowStore.val.value.modules[moduleIndex].value = { - ...flowStore.val.value.modules[moduleIndex].value, - content: codeToType - } - flowStore.val = { ...flowStore.val } - } - - // Press Enter after finishing typing - await wait(DELAY_MEDIUM) - const model = editor.getModel() - if (model && 'setValue' in model) { - model.setValue(currentText + '\n') - } - - // Mark step 4 as complete - step4Complete = true - } - } - }, - popover: { - title: 'Add validation logic', - description: 'Watch as we write code to validate the temperature input.', - side: 'bottom', - onNextClick: () => { - // Only proceed if code writing is complete - if (!step4Complete) { - sendUserToast( - 'Please wait for the code to finish typing...', - false, - [], - undefined, - 3000 - ) - return - } - - const driverOverlay = getDriverOverlay() - if (driverOverlay) { - driverOverlay.style.display = 'none' - } - - const customOverlay = document.createElement('div') - customOverlay.className = 'tutorial-custom-overlay' - customOverlay.style.cssText = ` - position: fixed; - top: 0; - left: 0; - width: 100%; - height: 100%; - background-color: rgba(0, 0, 0, 0.5); - z-index: 9999; - pointer-events: none; - clip-path: polygon( - 0 0, 100% 0, 100% 50%, 50% 50%, 50% 100%, 0 100% - ); - ` - document.body.appendChild(customOverlay) - - driver.moveNext() - }, - onPrevClick: () => { - sendUserToast('Previous is not available for this step', true, [], undefined, 3000) - } - } - }, - { - onHighlighted: async () => { - step5Complete = false - - // Create a single cursor that will move continuously - const fakeCursor = document.createElement('div') - fakeCursor.style.cssText = ` - position: fixed; - width: 20px; - height: 20px; - border-radius: 50%; - background-color: rgba(59, 130, 246, 0.8); - border: 2px solid white; - pointer-events: none; - z-index: 10000; - transition: all 1.5s ease-in-out; - ` - document.body.appendChild(fakeCursor) - - // Step 1: Move to and click plug button - document.querySelector('#flow-editor-plug')?.parentElement?.classList.remove('opacity-0') - await wait(DELAY_SHORT) - const plugButton = document.querySelector('#flow-editor-plug') as HTMLElement - if (plugButton) { - const plugRect = plugButton.getBoundingClientRect() - // Start from off-screen left - fakeCursor.style.left = `${plugRect.left - 100}px` - fakeCursor.style.top = `${plugRect.top + plugRect.height / 2}px` - await wait(DELAY_SHORT) - // Move to plug button - fakeCursor.style.left = `${plugRect.left + plugRect.width / 2}px` - fakeCursor.style.top = `${plugRect.top + plugRect.height / 2}px` - await wait(DELAY_ANIMATION) - await wait(DELAY_MEDIUM) - clickButtonBySelector('#flow-editor-plug') - } - - await wait(DELAY_MEDIUM) - - // Step 2: Move to and click flow_input.celsius - const targetButton = document.querySelector( - 'button[title="flow_input.celsius"]' - ) as HTMLElement - if (targetButton) { - await moveCursorToElement(fakeCursor, targetButton, DELAY_ANIMATION_LONG) - await wait(DELAY_MEDIUM) - const clickEvent = new MouseEvent('click', { - bubbles: true, - cancelable: true, - view: window - }) - targetButton.dispatchEvent(clickEvent) - } - - await wait(DELAY_LONG) - - // Step 3: Move to and click Test this step tab - const testTabButton = findButtonByText('Test this step', ['border-b-2', 'cursor-pointer']) - - if (testTabButton) { - await moveCursorToElement(fakeCursor, testTabButton, DELAY_ANIMATION) - await wait(DELAY_SHORT) - testTabButton.click() - } - - await wait(DELAY_LONG) - - // Step 4: Move to and click Run button - const testActionButton = findButtonByText('Run', ['bg-surface-accent-primary', 'w-full']) - - if (testActionButton) { - await moveCursorToElement(fakeCursor, testActionButton, DELAY_ANIMATION) - await wait(DELAY_MEDIUM) - testActionButton.click() - await wait(DELAY_MEDIUM) - } - - // Remove cursor at the end - fakeCursor.remove() - - step5Complete = true - }, - popover: { - title: 'Wire it up and test', - description: 'Connect the input, then run a quick test to verify the validation works.', - onNextClick: async () => { - if (!step5Complete) { - sendUserToast('Please wait for the test to complete...', false, [], undefined, 3000) - return - } - cleanupCustomOverlay() - driver.moveNext() - }, - onPrevClick: () => { - sendUserToast('Previous is not available for this step', true, [], undefined, 3000) - } - } - }, - { - onHighlighted: async () => { - step6Complete = false - - // First, add modules b and c with empty summaries - const modulesToAdd = [flowJson.value.modules[1], flowJson.value.modules[2]] - for (let i = 0; i < modulesToAdd.length; i++) { - await new Promise((resolve) => setTimeout(resolve, i === 0 ? 0 : 700)) - - const moduleData = modulesToAdd[i] - const module: FlowModule = { - id: moduleData.id, - summary: '', // Start with empty summary - value: moduleData.value - } - - await addModuleToFlow(module) - } - - await wait(700) - - // Create a single cursor for continuous movement - const fakeCursor = document.createElement('div') - fakeCursor.style.cssText = ` - position: fixed; - width: 20px; - height: 20px; - border-radius: 50%; - background-color: rgba(59, 130, 246, 0.8); - border: 2px solid white; - pointer-events: none; - z-index: 10000; - transition: all 1.5s ease-in-out; - ` - document.body.appendChild(fakeCursor) - - // Step 1: Click on script 'b' - await wait(DELAY_MEDIUM) - const scriptB = document.querySelector('#b') as HTMLElement - if (scriptB) { - const bRect = scriptB.getBoundingClientRect() - // Start from off-screen - fakeCursor.style.left = `${bRect.left - 100}px` - fakeCursor.style.top = `${bRect.top + bRect.height / 2}px` - await wait(DELAY_SHORT) - // Move to script b - fakeCursor.style.left = `${bRect.left + bRect.width / 2}px` - fakeCursor.style.top = `${bRect.top + bRect.height / 2}px` - await wait(DELAY_ANIMATION) - await wait(DELAY_MEDIUM) - selectionManager.selectId('b') - } - - await wait(DELAY_LONG) - - // Type summary for script 'b' - const summaryInputB = document.querySelector( - 'input[placeholder="Summary"]' - ) as HTMLInputElement - if (summaryInputB) { - const summaryTextB = 'Convert to Fahrenheit' - await typeText(summaryInputB, summaryTextB) - updateModuleSummary('b', summaryTextB) - await wait(DELAY_LONG) - } - - // Step 2: Move to and click on script 'c' - const scriptC = document.querySelector('#c') as HTMLElement - if (scriptC) { - await moveCursorToElement(fakeCursor, scriptC, DELAY_ANIMATION) - await wait(DELAY_SHORT) - selectionManager.selectId('c') - } - - await wait(DELAY_LONG) - - // Type summary for script 'c' - const summaryInputC = document.querySelector( - 'input[placeholder="Summary"]' - ) as HTMLInputElement - if (summaryInputC) { - const summaryTextC = 'Categorize temperature' - await typeText(summaryInputC, summaryTextC) - updateModuleSummary('c', summaryTextC) - await wait(DELAY_LONG) - } - - // Move cursor to Test Flow button - const testFlowButton = document.querySelector('#flow-editor-test-flow') as HTMLElement - if (testFlowButton) { - await moveCursorToElement(fakeCursor, testFlowButton, DELAY_ANIMATION) - await wait(DELAY_MEDIUM) - } - - // Remove cursor at the end - fakeCursor.remove() - - step6Complete = true - }, - popover: { - title: 'Add the final steps', - description: 'Two more scripts to convert and categorize the temperature.', - onNextClick: () => { - if (!step6Complete) { - sendUserToast( - 'Please wait for the summaries to be added...', - false, - [], - undefined, - 3000 - ) - return - } - - // Reset the driver.js overlay to full screen - const driverOverlay = getDriverOverlay() - if (driverOverlay) { - driverOverlay.style.display = '' - driverOverlay.style.width = '' - driverOverlay.style.right = '' - driverOverlay.style.left = '' - } - driver.moveNext() - }, - onPrevClick: () => { - sendUserToast('Previous is not available for this step', true, [], undefined, 3000) - } - } - }, - { - element: '#flow-editor-test-flow', - popover: { - title: 'Ready to test!', - description: - 'Run the complete flow and see your temperature converter in action.

💡 Want to learn more? Access more tutorials from the Tutorials page in the main menu or in the Help submenu.

', - onNextClick: () => { - updateProgress(index) - driver.destroy() - }, - onPrevClick: () => { - sendUserToast('Previous is not available for this step', true, [], undefined, 3000) - } - } - } - ] - - return steps - }} -/> diff --git a/frontend/src/lib/components/tutorials/OperatorTour.svelte b/frontend/src/lib/components/tutorials/OperatorTour.svelte new file mode 100644 index 0000000000..cc7f4ab4a5 --- /dev/null +++ b/frontend/src/lib/components/tutorials/OperatorTour.svelte @@ -0,0 +1,81 @@ + + + { + const steps: DriveStep[] = [ + { + popover: { + title: 'Welcome to Windmill! 🎉', + description: + "Let's take a quick tour! We'll show you the three main tools you can use: Scripts, Flows, and Apps." + } + }, + { + popover: { + title: 'Scripts - Run automated tasks', + description: + 'Script Example

Scripts are ready-to-use tasks that do things automatically for you.

You can run scripts whenever you need them - like generating a report, sending notifications, or processing data.

' + }, + element: '[data-value="script"]' + }, + { + popover: { + title: 'Flows - Run step-by-step processes', + description: + 'Flow

Flows are processes that run multiple tasks in order, one after another.

You can start a flow and watch it complete each step automatically - perfect for tasks that have multiple stages.

' + }, + element: '[data-value="flow"]' + }, + { + popover: { + title: 'Apps - Use custom tools', + description: + 'App

Apps are easy-to-use tools with buttons, forms, and displays built just for your team.

You can open an app to work with your data, fill out forms, or trigger tasks - no technical knowledge needed!

' + }, + element: '[data-value="app"]' + }, + { + popover: { + title: 'Finally, the Menu section', + description: + 'Explore available tabs where you can access your history of runs, your scheduled scripts, and your workspaces.

💡 Want to see this again? Pick Take the tour from that same menu.

', + onNextClick: async () => { + // The step points into the menu, so it has to be open before the popover + // lands on it — and open is also where the entry to re-run the tour is. + const menuButton = document.querySelector('[role="menuitem"]') as HTMLElement | null + menuButton?.click() + await wait(MENU_OPEN_DELAY_MS) + driver.destroy() + } + }, + element: '[role="menuitem"]' + } + ] + + return steps + }} +/> diff --git a/frontend/src/lib/components/tutorials/RunsTutorial.svelte b/frontend/src/lib/components/tutorials/RunsTutorial.svelte deleted file mode 100644 index 0b19ac5f0e..0000000000 --- a/frontend/src/lib/components/tutorials/RunsTutorial.svelte +++ /dev/null @@ -1,510 +0,0 @@ - - - { - return getTutorialSteps(driver) - }} -/> diff --git a/frontend/src/lib/components/tutorials/SkipTutorials.svelte b/frontend/src/lib/components/tutorials/SkipTutorials.svelte deleted file mode 100644 index 6fd6ba254b..0000000000 --- a/frontend/src/lib/components/tutorials/SkipTutorials.svelte +++ /dev/null @@ -1,32 +0,0 @@ - - -
- - -
diff --git a/frontend/src/lib/components/tutorials/TroubleshootFlowTutorial.svelte b/frontend/src/lib/components/tutorials/TroubleshootFlowTutorial.svelte deleted file mode 100644 index 4fa9909e02..0000000000 --- a/frontend/src/lib/components/tutorials/TroubleshootFlowTutorial.svelte +++ /dev/null @@ -1,441 +0,0 @@ - - - { - const steps: DriveStep[] = [ - { - popover: { - title: '🛠️ Troubleshoot a broken flow', - description: - 'We created a flow that is a temperature converter that validates input and converts Celsius to Fahrenheit. For this tutorial, our flow is intentionally broken.', - onNextClick: () => { - driver.moveNext() - } - } - }, - { - element: SELECTORS.testFlowButton, - onHighlighted: async () => { - stepComplete[1] = false - await wait(DELAY_SHORT) - stepComplete[1] = true - }, - popover: { - title: 'Test our flow', - description: - 'Let\'s run it so you can see what needs to be fixed.', - side: 'bottom', - onNextClick: async () => { - if (!checkStepComplete(1)) return - - // Click the Test Flow button to open the drawer - const testFlowButton = document.querySelector(SELECTORS.testFlowButton) as HTMLElement - if (testFlowButton) { - testFlowButton.click() - await wait(DELAY_LONG) - } - - driver.moveNext() - } - } - }, - { - element: SELECTORS.testFlowDrawer, - onHighlighted: async () => { - stepComplete[2] = false - await wait(DELAY_SHORT) - stepComplete[2] = true - }, - popover: { - title: 'Run the flow', - description: - 'Click "Next" to execute the flow. We\'ll use the results to troubleshoot the error.', - side: 'left', - onNextClick: async () => { - if (!checkStepComplete(2)) return - - // Click the Test button to execute the flow - const testButton = document.querySelector(SELECTORS.testFlowDrawer) as HTMLElement - if (testButton) { - testButton.click() - } - - await wait(DELAY_LONG) - driver.moveNext() - } - } - }, - { - element: '.border.rounded-md.shadow.p-2', - onHighlighted: async () => { - stepComplete[3] = false - await wait(DELAY_SHORT) - stepComplete[3] = true - }, - popover: { - title: 'Review the error', - description: - 'Our flow failed. Let\'s review the error and understand what happened.', - side: 'left', - onNextClick: () => { - if (!checkStepComplete(3)) return - driver.moveNext() - } - } - }, - { - element: '.border-b.flex.flex-row.whitespace-nowrap.scrollbar-hidden.mx-auto', - onHighlighted: async () => { - stepComplete[4] = false - await wait(DELAY_SHORT) - stepComplete[4] = true - }, - popover: { - title: 'Explore the tabs', - description: - 'Use these tabs to navigate between different views: Result, Logs, and Graph. We\'ll focus on the Graph tab to review the error.', - side: 'bottom', - onNextClick: () => { - if (!checkStepComplete(4)) return - driver.moveNext() - } - } - }, - { - element: '.grid.grid-cols-3.border.h-full', - onHighlighted: async () => { - stepComplete[5] = false - await wait(DELAY_SHORT) - - // Find the step 'b' button inside the drawer and click it with fake cursor - const flowPreviewContent = getElementBySelector(SELECTORS.flowPreviewContent) - if (flowPreviewContent) { - const stepButton = findButtonByText(flowPreviewContent, TEXT.convertToFahrenheit) - - if (stepButton) { - await animateFakeCursorClick(stepButton, 1.5, { usePointerEvents: true }) - await wait(DELAY_MEDIUM) - } - } - - stepComplete[5] = true - }, - popover: { - title: 'Inspect the flow graph', - description: - 'B step failed during the run. Let\'s take a closer look at its behavior.', - side: 'top', - onNextClick: () => { - if (!checkStepComplete(5)) return - driver.moveNext() - } - } - }, - { - element: '.rounded-md.grow.bg-surface-tertiary.text-xs.flex.flex-col.max-h-screen.gap-2.overflow-hidden.border', - onHighlighted: async () => { - stepComplete[6] = false - await wait(DELAY_SHORT) - stepComplete[6] = true - }, - popover: { - title: 'Error spotted!', - description: - 'We made a typo in the code. Let\'s fix it and run the flow again.', - side: 'left', - onNextClick: async () => { - if (!checkStepComplete(6)) return - - // Click the close button inside the drawer - const drawer = getElementBySelector(SELECTORS.flowPreviewContent) - if (drawer) { - const closeButton = findCloseButton(drawer) - - if (closeButton) { - await animateFakeCursorClick(closeButton, 1.5) - } - } - - await wait(DELAY_LONG) - driver.moveNext() - } - } - }, - { - element: SELECTORS.stepB, - onHighlighted: async () => { - stepComplete[7] = false - await wait(DELAY_SHORT) - - // Click on div id="b" to open the editor - const stepBDiv = getElementBySelector(SELECTORS.stepB) - if (stepBDiv) { - await animateFakeCursorClick(stepBDiv, 1.5) - await wait(DELAY_LONG) - } - - stepComplete[7] = true - }, - popover: { - title: 'Your turn now!', - description: - 'Fix the issue in the code, and run the flow again to confirm everything works.

💡 Want to learn more? Access more tutorials from the Tutorials page in the main menu or in the Help submenu.

', - side: 'top', - onNextClick: () => { - if (!checkStepComplete(7)) return - updateProgress(index) - driver.destroy() - } - } - } - ] - - return steps - }} -/> diff --git a/frontend/src/lib/components/tutorials/Tutorial.svelte b/frontend/src/lib/components/tutorials/Tutorial.svelte index fc95141f5c..d4c99482a2 100644 --- a/frontend/src/lib/components/tutorials/Tutorial.svelte +++ b/frontend/src/lib/components/tutorials/Tutorial.svelte @@ -1,155 +1,101 @@ {#if tutorial} diff --git a/frontend/src/lib/components/tutorials/TutorialControls.svelte b/frontend/src/lib/components/tutorials/TutorialControls.svelte index 826e66d151..dd15cf4151 100644 --- a/frontend/src/lib/components/tutorials/TutorialControls.svelte +++ b/frontend/src/lib/components/tutorials/TutorialControls.svelte @@ -1,51 +1,39 @@
{#if activeIndex === 0} -
  • UI is not interactive during tutorial, press next at every step
    • -
  • You can use the arrow keys to navigate
    • +
  • UI is not interactive during the tour, press next at every step
    • +
  • You can use the arrow keys to navigate
    •  {/if}
    - {#if activeIndex !== undefined && totalSteps !== undefined} -
    - Step {activeIndex + 1} of {totalSteps} -
    - {/if} +
    + Step {activeIndex + 1} of {totalSteps} +
    -
    diff --git a/frontend/src/lib/components/tutorials/TutorialProgressBar.svelte b/frontend/src/lib/components/tutorials/TutorialProgressBar.svelte deleted file mode 100644 index 5084299a7e..0000000000 --- a/frontend/src/lib/components/tutorials/TutorialProgressBar.svelte +++ /dev/null @@ -1,29 +0,0 @@ - - -
    -
    -
    - Progress: {completed} of {total} {label} completed -
    -
    {progressPercentage}%
    -
    -
    -
    -
    -
    - diff --git a/frontend/src/lib/components/tutorials/TutorialRouter.svelte b/frontend/src/lib/components/tutorials/TutorialRouter.svelte deleted file mode 100644 index 80c8938eff..0000000000 --- a/frontend/src/lib/components/tutorials/TutorialRouter.svelte +++ /dev/null @@ -1,64 +0,0 @@ - - -{#each tutorials as tutorial} - -{/each} - diff --git a/frontend/src/lib/components/tutorials/TutorialWrapper.svelte b/frontend/src/lib/components/tutorials/TutorialWrapper.svelte deleted file mode 100644 index 32b6fa212f..0000000000 --- a/frontend/src/lib/components/tutorials/TutorialWrapper.svelte +++ /dev/null @@ -1,36 +0,0 @@ - - -{#if Component} - {@const Comp = Component} - -{/if} - diff --git a/frontend/src/lib/components/tutorials/app/BackgroundRunnablesTutorial.svelte b/frontend/src/lib/components/tutorials/app/BackgroundRunnablesTutorial.svelte deleted file mode 100644 index 65fa99ed46..0000000000 --- a/frontend/src/lib/components/tutorials/app/BackgroundRunnablesTutorial.svelte +++ /dev/null @@ -1,91 +0,0 @@ - - - { - const steps: DriveStep[] = [ - { - element: '#app-editor-runnable-panel', - popover: { - title: 'Runnable panel', - description: - 'This is the runnable panel. Here you can add runnables to your app. Runnables are scripts that can be executed in the background. You can add as many runnables as you want.' - } - }, - { - element: '#create-background-runnable', - popover: { - title: 'Create a runnable', - description: - 'Click here to create a runnable. Runnables are scripts that can be executed in the background. You can add as many runnables as you want.', - onNextClick: () => { - clickButtonBySelector('#create-background-runnable') - setTimeout(() => driver.moveNext()) - } - } - }, - { - element: '#app-editor-empty-runnable', - popover: { - title: 'Empty runnable panel', - description: - 'This is the empty runnable panel. Here you can add runnables to your app. Runnables are scripts that can be executed in the background. You can add as many runnables as you want. You can also select a script or a flow from your workspace or the Hub.' - } - }, - - { - element: '#app-editor-backend-runnables', - popover: { - title: 'Backend runnables', - description: - 'Backend runnables are scripts that are executed on the server. They can be used to perform tasks that are not possible to be performed on the client. For example, you can use backend runnables to send emails, perform database operations, etc.' - } - }, - { - element: '#app-editor-frontend-runnables', - popover: { - title: 'Frontend runnables', - description: - 'Frontend scripts are executed in the browser and can manipulate the app context directly. You can also interact with components using component controls.', - onNextClick: () => { - setTimeout(() => { - driver.moveNext() - - updateProgress(index) - }) - } - } - } - ] - - // Remove steps if we want to skip them (excpet the first one) - - if (options?.skipStepsCount) { - steps.splice(1, options.skipStepsCount) - } - - return steps - }} -/> diff --git a/frontend/src/lib/components/tutorials/app/ConnectionTutorial.svelte b/frontend/src/lib/components/tutorials/app/ConnectionTutorial.svelte deleted file mode 100644 index 9ae02af92a..0000000000 --- a/frontend/src/lib/components/tutorials/app/ConnectionTutorial.svelte +++ /dev/null @@ -1,128 +0,0 @@ - - - [ - { - popover: { - title: 'Connection tutorial', - description: 'We will connect the input of a text component to an output.', - onNextClick: () => { - addComponent() - setTimeout(() => { - driver.moveNext() - }) - } - } - }, - { - element: `#component-input`, - popover: { - title: 'Data source', - description: - 'Here we can set the data source of the text component: it can be static, the result of an evaluation or the result of script or flow. We are going to connect the data source to an output.', - onNextClick: () => { - clickButtonBySelector('#component-input') - setTimeout(() => { - driver.moveNext() - }) - } - } - }, - { - element: '[data-connection-button] button[title="Connect"]', - popover: { - title: 'Connect the text component', - description: 'Click on the plug icon to connect the text component', - onNextClick: () => { - clickButtonBySelector('[data-connection-button] button[title="Connect"]') - setTimeout(() => { - driver.moveNext() - }) - } - } - }, - { - element: '#output-ctx', - popover: { - title: 'Select the output', - description: - "You can now select the output in the output menu. Let's select your email in the app context", - onNextClick: () => { - clickButtonBySelector('#output-ctx') - setTimeout(() => { - driver.moveNext() - }) - } - } - }, - { - element: '.val', - popover: { - title: 'Click on the output', - description: 'Simply click on the output to connect it', - onNextClick: () => { - clickButtonBySelector('.val') - setTimeout(() => { - driver.moveNext() - }) - } - } - }, - { - popover: { - title: 'Connection done', - description: 'You can now see the email output connected to the text component input', - onNextClick: () => { - updateProgress(index) - - setTimeout(() => { - driver.moveNext() - }) - } - } - } - ]} -/> diff --git a/frontend/src/lib/components/tutorials/app/ExpressionEvaluationTutorial.svelte b/frontend/src/lib/components/tutorials/app/ExpressionEvaluationTutorial.svelte deleted file mode 100644 index 1e96e8b121..0000000000 --- a/frontend/src/lib/components/tutorials/app/ExpressionEvaluationTutorial.svelte +++ /dev/null @@ -1,33 +0,0 @@ - - - [ - { - popover: { - title: 'Expression evaluation tutorial', - description: - 'Learn how to build our first branch to be executed on a condition. You can use arrow keys to navigate' - } - } - ]} -/> diff --git a/frontend/src/lib/components/tutorials/ignoredTutorials.ts b/frontend/src/lib/components/tutorials/ignoredTutorials.ts deleted file mode 100644 index 7a120b5e0c..0000000000 --- a/frontend/src/lib/components/tutorials/ignoredTutorials.ts +++ /dev/null @@ -1,3 +0,0 @@ -import { writable } from 'svelte/store' - -export const ignoredTutorials = writable([]) diff --git a/frontend/src/lib/components/tutorials/operatorTour.ts b/frontend/src/lib/components/tutorials/operatorTour.ts new file mode 100644 index 0000000000..5c4269e84d --- /dev/null +++ b/frontend/src/lib/components/tutorials/operatorTour.ts @@ -0,0 +1,47 @@ +import { UserService } from '$lib/gen' + +/** + * The tour's slot in the `tutorial_progress` bitmask. Slot 6 is reserved for it across + * versions: an operator who has already been through the tour must not meet it again, and + * a slot that another tutorial writes would read as finished on day one. + */ +const OPERATOR_TOUR_BIT = 6 + +/** URL parameter the sidebar entry uses to ask the home page for a run. */ +export const TOUR_PARAM = 'tour' +export const TOUR_PARAM_VALUE = 'operator' + +/** Long enough for the home page's tabs to exist before the first step points at one. */ +export const TOUR_START_DELAY_MS = 500 +/** Time for the sidebar to open before the last step points into it. */ +export const MENU_OPEN_DELAY_MS = 300 + +export async function hasSeenOperatorTour(): Promise { + // A failure answers "seen": the tour interrupts the page, and interrupting someone who + // has already been through it is worse than never offering it, which the sidebar entry + // covers anyway. + try { + const progress = (await UserService.getTutorialProgress()).progress ?? 0 + return (progress & (1 << OPERATOR_TOUR_BIT)) !== 0 + } catch (error) { + console.error('Could not read tutorial progress:', error) + return true + } +} + +export async function markOperatorTourSeen(): Promise { + try { + // Read-modify-write, because the row is shared: it carries every slot's state, and a + // write of this bit alone would clear the rest. `skipped_all` rides along for the same + // reason — and the handler rejects a body without it, whatever the generated type says. + const current = await UserService.getTutorialProgress() + await UserService.updateTutorialProgress({ + requestBody: { + progress: (current.progress ?? 0) | (1 << OPERATOR_TOUR_BIT), + skipped_all: current.skipped_all ?? false + } + }) + } catch (error) { + console.error('Could not record tutorial progress:', error) + } +} diff --git a/frontend/src/lib/components/tutorials/utils.ts b/frontend/src/lib/components/tutorials/utils.ts deleted file mode 100644 index 083e9712dd..0000000000 --- a/frontend/src/lib/components/tutorials/utils.ts +++ /dev/null @@ -1,328 +0,0 @@ -import type { FlowModule, OpenFlow } from '$lib/gen' -import { deepEqual } from 'fast-equals' -import { emptyApp } from '../apps/editor/appUtils' -import type { App } from '../apps/types' -import { findGridItem } from '../apps/editor/appUtilsCore' -import { isRunnableByName } from '../apps/inputType' -import { wait } from '$lib/utils' - -// Tutorial animation delay constants -export const DELAY_SHORT = 100 -export const DELAY_MEDIUM = 300 -export const DELAY_LONG = 500 -export const DELAY_ANIMATION = 1500 -export const DELAY_ANIMATION_LONG = 2500 -export const DELAY_TYPING = 50 -export const DELAY_CODE_CHAR = 2 -export const DELAY_CODE_NEWLINE = 5 - -export function setInputBySelector(selector: string, value: string) { - const input = document.querySelector(selector) as HTMLInputElement - - if (input) { - input.value = value - input.dispatchEvent(new Event('input', { bubbles: true })) - } -} - -export function clickButtonBySelector(selector: string) { - const button = document.querySelector(selector) as HTMLButtonElement - - if (button) { - button.click() - } -} - -export function clickFirstButtonBySelector(selector: string) { - const buttons = document.querySelector(selector) - const button = buttons?.childNodes[0] as HTMLButtonElement - - if (button) { - button.click() - } -} - -export function triggerPointerDown(selector: string) { - const elem = document.querySelector(selector) as HTMLElement - - if (elem) { - elem.dispatchEvent(new PointerEvent('pointerdown', { bubbles: true })) - } -} - -export function selectOptionsBySelector(selector: string, value: string) { - const select = document.querySelector(selector) as HTMLSelectElement - - if (select) { - select.value = value - select.dispatchEvent(new Event('change', { bubbles: true })) - } -} - -export function isFlowTainted(flow: OpenFlow) { - return ( - flow.value.modules.length > 0 || Object.keys((flow?.schema?.properties as any) ?? {}).length > 0 - ) -} - -export function isAppTainted(app: App) { - if (app.hideLegacyTopBar === true) { - // An empty app should have only have a topbar and no hidden inline scripts - - if (Array.isArray(app.hiddenInlineScripts) && app.hiddenInlineScripts?.length > 0) { - return true - } - - // New apps have only a single component which is the topbar - if (Array.isArray(app.grid) && app.grid.length > 1) { - return true - } - - // Check if the current app is different from an empty app - return !deepEqual(app, emptyApp()) - } else { - // For older apps, - return !(app.grid?.length === 0 && app.hiddenInlineScripts?.length === 0) - } -} - -export function updateFlowModuleById( - flow: OpenFlow, - id: string, - callback: (module: FlowModule) => void -) { - const dfs = (modules: FlowModule[]) => { - for (const module of modules) { - if (module.id === id) { - callback(module) - return - } - - if (module.value.type === 'forloopflow') { - dfs(module.value.modules) - } else if (module.value.type === 'branchone') { - module.value.branches.forEach((branch) => dfs(branch.modules)) - } else if (module.value.type === 'branchall') { - module.value.branches.forEach((branch) => dfs(branch.modules)) - } - // AI agent tools are leaf nodes - no traversal needed - } - } - - dfs(flow.value.modules) -} - -export function updateBackgroundRunnableCode(app: App, index: number, newCode: string) { - const script = app.hiddenInlineScripts[index] - if (isRunnableByName(script) && script.inlineScript) { - script.inlineScript.content = newCode - } -} - -export function updateInlineRunnableCode(app: App, componentId: string, newCode: string) { - const gridItem = findGridItem(app, componentId) - if (gridItem?.data.componentInput?.type === 'runnable') { - if ( - isRunnableByName(gridItem.data.componentInput.runnable) && - gridItem.data.componentInput.runnable.inlineScript - ) { - gridItem.data.componentInput.runnable.inlineScript.content = newCode - } - } -} - -export function connectComponentSourceToOutput(app: App, componentId: string, targetId: string) { - const gridItem = findGridItem(app, componentId) - - if (gridItem) { - gridItem.data.componentInput = { - type: 'evalv2', - fieldType: 'object', - - expr: `${targetId}.result`, - connections: [ - { - componentId: targetId, - id: 'result' - } - ] - } - } -} - -export function connectInlineRunnableInputToComponentOutput( - app: App, - sourceComponentId: string, - sourceField: string, - targetComponentId: string, - targetField: string, - fieldType: string = 'text' -) { - const gridItem = findGridItem(app, sourceComponentId) - - if (gridItem?.data.componentInput?.type === 'runnable') { - // @ts-ignore - gridItem.data.componentInput.fields = { - [sourceField]: { - type: 'evalv2', - expr: `${targetComponentId}.${targetField}`, - fieldType: fieldType, - connections: [ - { - componentId: targetComponentId, - id: targetField - } - ] - } - } - } -} - -function elementExists(selector: string): boolean { - return !!document.querySelector(selector) -} - -export function waitForElementLoading( - selector: string, - callback: () => void, - interval: number = 50, - maxAttempts: number = 30 -): void { - let attempts = 0 - - const checkExistence = setInterval(() => { - if (elementExists(selector)) { - clearInterval(checkExistence) - callback() - } else if (attempts >= maxAttempts) { - clearInterval(checkExistence) - console.error('Element not found after multiple attempts.') - } - attempts++ - }, interval) -} - -// Helper function to move cursor to element (for continuous cursor movement in tutorials) -export async function moveCursorToElement( - cursor: HTMLElement, - element: HTMLElement, - duration: number = DELAY_ANIMATION -): Promise { - const rect = element.getBoundingClientRect() - cursor.style.transition = `all ${duration / 1000}s ease-in-out` - cursor.style.left = `${rect.left + rect.width / 2}px` - cursor.style.top = `${rect.top + rect.height / 2}px` - await wait(duration) -} - -// Helper function to create a fake cursor element for tutorial animations -export function createFakeCursor(): HTMLElement { - const fakeCursor = document.createElement('div') - fakeCursor.style.cssText = ` - position: fixed; - width: 20px; - height: 20px; - border-radius: 50%; - background-color: rgba(59, 130, 246, 0.8); - border: 2px solid white; - pointer-events: none; - z-index: 10000; - transition: all 1.5s ease-in-out; - ` - document.body.appendChild(fakeCursor) - return fakeCursor -} - -// Constants for cursor animation -const CURSOR_START_OFFSET = -100 -const CURSOR_CLICK_SCALE = 0.8 - -// Helper function to create and animate a fake cursor with start position -export async function createFakeCursorWithStart( - startElement: HTMLElement | null, - endElement: HTMLElement, - transitionDuration: number = 1.5 -): Promise { - const fakeCursor = createFakeCursor() - - const endRect = endElement.getBoundingClientRect() - let startX: number, startY: number - - if (startElement) { - const startRect = startElement.getBoundingClientRect() - startX = startRect.left + startRect.width / 2 - startY = startRect.top + startRect.height / 2 - } else { - startX = endRect.left + CURSOR_START_OFFSET - startY = endRect.top + endRect.height / 2 - } - - fakeCursor.style.left = `${startX}px` - fakeCursor.style.top = `${startY}px` - - await wait(DELAY_SHORT) - - fakeCursor.style.left = `${endRect.left + endRect.width / 2}px` - fakeCursor.style.top = `${endRect.top + endRect.height / 2}px` - - await wait(transitionDuration * 1000) - - return fakeCursor -} - -// Helper function to animate a fake cursor click -export async function animateFakeCursorClick( - element: HTMLElement, - transitionDuration: number = 1.5, - options?: { usePointerEvents?: boolean; startElement?: HTMLElement | null } -): Promise { - const fakeCursor = await createFakeCursorWithStart( - options?.startElement ?? null, - element, - transitionDuration - ) - await wait(DELAY_MEDIUM) - - // Animate click (shrink cursor briefly) - fakeCursor.style.transform = `scale(${CURSOR_CLICK_SCALE})` - await wait(DELAY_SHORT) - fakeCursor.style.transform = 'scale(1)' - await wait(DELAY_SHORT) - - // Trigger pointer events if needed (flow graph uses pointer events instead of click) - if (options?.usePointerEvents) { - element.dispatchEvent(new PointerEvent('pointerdown', { bubbles: true })) - element.dispatchEvent(new PointerEvent('pointerup', { bubbles: true })) - } - - // Click the element - element.click() - await wait(DELAY_SHORT) - - // Remove fake cursor - fakeCursor.remove() -} - -// Helper function to animate cursor to element and click (for reusing a cursor across multiple clicks) -export async function animateCursorToElementAndClick( - cursor: HTMLElement, - element: HTMLElement, - startOffset: number = CURSOR_START_OFFSET -): Promise { - const rect = element.getBoundingClientRect() - - // Set initial position (off-screen to the left) - cursor.style.left = `${rect.left + startOffset}px` - cursor.style.top = `${rect.top + rect.height / 2}px` - await wait(DELAY_SHORT) - - // Animate to target position - cursor.style.left = `${rect.left + rect.width / 2}px` - cursor.style.top = `${rect.top + rect.height / 2}px` - await wait(DELAY_ANIMATION) - await wait(DELAY_MEDIUM) - - // Click on the element - element.click() - await wait(DELAY_SHORT) -} diff --git a/frontend/src/lib/components/tutorials/workspace/WorkspaceOnboardingOperatorTutorial.svelte b/frontend/src/lib/components/tutorials/workspace/WorkspaceOnboardingOperatorTutorial.svelte deleted file mode 100644 index 4251a67170..0000000000 --- a/frontend/src/lib/components/tutorials/workspace/WorkspaceOnboardingOperatorTutorial.svelte +++ /dev/null @@ -1,141 +0,0 @@ - - - { - const steps: DriveStep[] = [ - { - popover: { - title: 'Welcome to Windmill! 🎉', - description: - "Let's take a quick tour! We'll show you the three main tools you can use: Scripts, Flows, and Apps.", - onNextClick: () => { - // Wait a bit to ensure the page is fully rendered before moving to next step - setTimeout(() => { - // Try to find the script tab button - const scriptsButton = document.querySelector('[data-value="script"]') as HTMLElement | null - - if (scriptsButton) { - driver.moveNext() - } else { - // If we can't find the button, just move to next step anyway - driver.moveNext() - } - }, 100) - } - } - }, - { - popover: { - title: 'Scripts - Run automated tasks', - description: - 'Script Example

    Scripts are ready-to-use tasks that do things automatically for you.

    You can run scripts whenever you need them - like generating a report, sending notifications, or processing data.

    ', - onNextClick: async () => { - // Move to the next step (Flows) - setTimeout(() => { - const flowsButton = document.querySelector('[data-value="flow"]') as HTMLElement | null - - if (flowsButton) { - driver.moveNext() - } else { - driver.moveNext() - } - }, 100) - } - }, - element: '[data-value="script"]' - }, - { - popover: { - title: 'Flows - Run step-by-step processes', - description: - 'Flow

    Flows are processes that run multiple tasks in order, one after another.

    You can start a flow and watch it complete each step automatically - perfect for tasks that have multiple stages.

    ', - onNextClick: async () => { - // Move to the next step (Apps) - setTimeout(() => { - const appsButton = document.querySelector('[data-value="app"]') as HTMLElement | null - - if (appsButton) { - driver.moveNext() - } else { - driver.moveNext() - } - }, 100) - } - }, - element: '[data-value="flow"]' - }, - { - popover: { - title: 'Apps - Use custom tools', - description: - 'App

    Apps are easy-to-use tools with buttons, forms, and displays built just for your team.

    You can open an app to work with your data, fill out forms, or trigger tasks - no technical knowledge needed!

    ', - onNextClick: async () => { - // Move to the next step (cursor animation) - driver.moveNext() - } - }, - element: '[data-value="app"]' - }, - { - popover: { - title: 'Finally, the Menu section', - description: 'Explore available tabs where you can access your history of runs, your scheduled scripts, your tutorials progress etc.

    💡 Want to learn more? Access more tutorials from the Tutorials page in the main menu.

    ', - onNextClick: async () => { - // Find the target button and click it - const targetButton = document.querySelector('[role="menuitem"]') as HTMLElement | null - if (targetButton) { - targetButton.click() - } - - // Wait for menu to open - await wait(DELAY_MEDIUM) - - // Mark tutorial as complete - updateProgress(index) - driver.destroy() - - // Clean up URL parameter if present - if (page.url.searchParams.has('tutorial')) { - goto(`${base}/`, { replaceState: true }) - } - } - }, - element: '[role="menuitem"]' - } - ] - - return steps - }} -/> diff --git a/frontend/src/lib/components/tutorials/workspace/WorkspaceOnboardingTutorial.svelte b/frontend/src/lib/components/tutorials/workspace/WorkspaceOnboardingTutorial.svelte deleted file mode 100644 index 30307463a1..0000000000 --- a/frontend/src/lib/components/tutorials/workspace/WorkspaceOnboardingTutorial.svelte +++ /dev/null @@ -1,95 +0,0 @@ - - - { - const steps: DriveStep[] = [ - { - popover: { - title: 'Welcome to your Windmill workspace! 🎉', - description: - "Let's take a quick tour! We will show you the main sections of your workspace.", - onNextClick: async () => { - // The New menu button mounts once an async permission check resolves, so - // wait for it before highlighting it in the next step. - for (let i = 0; i < 20 && !document.querySelector('#create-new-button'); i++) { - await new Promise((resolve) => setTimeout(resolve, 100)) - } - driver.moveNext() - } - } - }, - { - popover: { - title: 'Create your first script', - description: - 'Programming Languages

    Open the New menu to create a script. Scripts turn code into tools. Write in Python, TypeScript, Go, Bash, SQL and more. Run them manually, on schedule, or via webhooks.

    ', - onNextClick: () => { - driver.moveNext() - } - }, - element: '#create-new-button' - }, - { - popover: { - title: 'Create your first flow', - description: - 'Flow

    The same New menu lets you create a flow. Flows orchestrate multiple scripts. Chain them together with branching, loops, and error handling to build complex workflows.

    ', - onNextClick: () => { - driver.moveNext() - } - }, - element: '#create-new-button' - }, - { - popover: { - title: 'Create your first app', - description: - 'App

    And from the New menu you can also create an app. Apps are custom UIs built with drag-and-drop. Combine tables, forms, charts, and buttons that trigger your scripts and flows. That\'s it for the tour!

    💡 Want to learn more? Access more tutorials from the Tutorials page in the main menu or in the Help submenu.

    ', - onNextClick: async () => { - // Mark tutorial as complete - updateProgress(index) - driver.destroy() - - // Clean up URL parameter if present - if (page.url.searchParams.has('tutorial')) { - goto(`${base}/`, { replaceState: true }) - } - } - }, - element: '#create-new-button' - } - ] - - return steps - }} -/> diff --git a/frontend/src/lib/components/workspaceSettings/SimpleCreateWorkspace.svelte b/frontend/src/lib/components/workspaceSettings/SimpleCreateWorkspace.svelte new file mode 100644 index 0000000000..67059c2fb8 --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/SimpleCreateWorkspace.svelte @@ -0,0 +1,237 @@ + + +{#if creating} +
    + + Creating {name.trim()}… +
    +{:else if advanced} + + + {#if leading} +
    {@render leading()}
    + {/if} +{:else} +
    + Workspace name + (nameEdited = true), + onkeydown: (e) => e.key === 'Enter' && create() + }} + /> + {#if problem && name.trim()} + {problem} + {/if} + {#if policyFailed} + + This instance's settings could not be read, so a workspace cannot be created yet. + + + {/if} + +
    +
    + {@render leading?.()} + + + +
    + +
    +
    +{/if} diff --git a/frontend/src/lib/hubProject.test.ts b/frontend/src/lib/hubProject.test.ts new file mode 100644 index 0000000000..ca1891110d --- /dev/null +++ b/frontend/src/lib/hubProject.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, it, vi } from 'vitest' + +vi.mock('./gen', () => ({ HubPublishService: {}, SettingService: {} })) +vi.mock('./components/icons', () => ({ appIconComponent: () => undefined })) + +import { hubProjectDescription } from './hubProject' + +describe('hubProjectDescription', () => { + it('prefers the description field when the hub has one', () => { + expect(hubProjectDescription({ description: ' Runs payroll. ', readme: '# Other' })).toBe( + 'Runs payroll.' + ) + }) + + it('reads the readme intro when it does not, which is every published project', () => { + expect( + hubProjectDescription({ + description: '', + readme: 'Audiences and campaigns,\nwith a sending engine.\n\n## Concepts\n\n- A flow' + }) + ).toBe('Audiences and campaigns, with a sending engine.') + }) + + it('skips a leading heading rather than stopping at it', () => { + expect( + hubProjectDescription({ + readme: '## Description\n\nManages Odoo records.\n\n## Usage\n\n1. Generate a key' + }) + ).toBe('Manages Odoo records.') + }) + + it('strips inline markdown', () => { + expect( + hubProjectDescription({ readme: 'A **bold** clone of [Bitly](https://bitly.com) with `js`.' }) + ).toBe('A bold clone of Bitly with js.') + }) + + it('cuts on a word boundary, so a long one reads as shortened not corrupted', () => { + const long = hubProjectDescription({ readme: 'lorem ipsum '.repeat(40) }) + expect(long.length).toBeLessThanOrEqual(321) + expect(long.endsWith('…')).toBe(true) + expect(long).not.toMatch(/lore…$/) + }) + + it('falls back to the summary when there is no prose at all', () => { + expect(hubProjectDescription({ readme: '## Usage\n', summary: 'Short links' })).toBe( + 'Short links' + ) + expect(hubProjectDescription({})).toBe('') + }) +}) diff --git a/frontend/src/lib/hubProject.ts b/frontend/src/lib/hubProject.ts index b58b338f7d..be108b2136 100644 --- a/frontend/src/lib/hubProject.ts +++ b/frontend/src/lib/hubProject.ts @@ -1,6 +1,6 @@ import type { Component } from 'svelte' import { appIconComponent } from '$lib/components/icons' -import { SettingService } from '$lib/gen' +import { HubPublishService, SettingService } from '$lib/gen' import { DEFAULT_HUB_BASE_URL } from '$lib/hub' import type { ImportProjectSummary } from '$lib/components/ImportProjectCard.svelte' @@ -85,3 +85,123 @@ const HUB_APP_ICON_ALIAS: Record = { postgres: 'postgresql' } export function hubAppIcon(app: string): Component | undefined { return appIconComponent(HUB_APP_ICON_ALIAS[app] ?? app) } + +/** One row of the hub's catalogue (`GET /projects`), which carries no item counts. */ +interface HubProjectListRow { + slug: string + name: string + summary: string + description: string + readme: string + author: string + apps: string[] + hasLogo: boolean + stars: number +} + +const DESCRIPTION_MAX = 320 + +/** + * What a project says about itself, in prose. + * + * The hub's `description` field is empty on every published project — the writing all + * goes in the readme — so the readme's opening paragraphs stand in. Everything from the + * first heading onwards is dropped: that is the "Windmill concepts demonstrated" / + * "Usage" material, which is documentation rather than a description. A readme that + * *starts* with a heading (`## Description`) has it skipped rather than treated as the + * end of the intro. + */ +export function hubProjectDescription(row: { + description?: string + readme?: string + summary?: string +}): string { + if (row.description?.trim()) return row.description.trim() + + const lines = (row.readme ?? '').split('\n') + let i = 0 + while (i < lines.length && (lines[i].trim() === '' || lines[i].startsWith('#'))) i++ + const intro: string[] = [] + for (; i < lines.length; i++) { + if (lines[i].startsWith('#')) break + intro.push(lines[i]) + } + + const text = intro + .join(' ') + // Inline markdown only — the block syntax is already gone with the headings. + .replace(/\[([^\]]+)\]\([^)]*\)/g, '$1') + .replace(/[*_`]/g, '') + .replace(/\s+/g, ' ') + .trim() + if (!text) return row.summary?.trim() ?? '' + if (text.length <= DESCRIPTION_MAX) return text + // Cut on a word boundary: a description sliced mid-word reads as corrupted rather + // than shortened. + const cut = text.slice(0, DESCRIPTION_MAX) + const lastSpace = cut.lastIndexOf(' ') + return `${(lastSpace > DESCRIPTION_MAX * 0.6 ? cut.slice(0, lastSpace) : cut).trimEnd()}…` +} + +/** + * A card in the template picker. Everything it shows comes from the catalogue listing, + * so a whole page of cards costs one request; the item counts, which only the import + * step needs, are fetched per project by `fetchHubProject` when one is picked. + * `id` is what `InfiniteList` dedupes rows by. + */ +export interface HubProjectPick { + id: string + slug: string + name: string + summary: string + description: string + author: string + apps: string[] + logoUrl?: string + iconApps: string[] + stars: number +} + +let catalogue: { workspace: string; projects: Promise } | undefined + +/** + * Every published project, most-starred first, fetched once per workspace and held for + * the life of the page. + * + * Through the workspace-scoped proxy rather than straight at the hub the way + * `fetchHubProject` goes: the catalogue endpoint sends no `Access-Control-Allow-Origin`, + * so the browser cannot read it directly. + */ +export function hubProjectCatalogue(workspace: string): Promise { + if (catalogue?.workspace !== workspace) { + const projects = loadCatalogue(workspace).catch((e) => { + // A cached rejection would make the failure permanent for the whole session; + // dropping it lets the next open try again. + if (catalogue?.projects === projects) catalogue = undefined + throw e + }) + catalogue = { workspace, projects } + } + return catalogue.projects +} + +async function loadCatalogue(workspace: string): Promise { + const raw = await HubPublishService.listHubProjects({ workspace }) + const rows = ((typeof raw === 'string' ? JSON.parse(raw) : raw)?.projects ?? + []) as HubProjectListRow[] + const hub = await hubBrowserUrl() + return rows + .map((row) => ({ + id: row.slug, + slug: row.slug, + name: row.name, + summary: row.summary, + description: hubProjectDescription(row), + author: row.author, + apps: row.apps ?? [], + logoUrl: row.hasLogo ? `${hub}/projects/${encodeURIComponent(row.slug)}/logo` : undefined, + iconApps: row.apps ?? [], + stars: row.stars ?? 0 + })) + .sort((a, b) => b.stars - a.stars || a.name.localeCompare(b.name)) +} diff --git a/frontend/src/lib/importWizard/abandon.test.ts b/frontend/src/lib/importWizard/abandon.test.ts index abf0ff61b9..d240c8147d 100644 --- a/frontend/src/lib/importWizard/abandon.test.ts +++ b/frontend/src/lib/importWizard/abandon.test.ts @@ -150,6 +150,30 @@ describe('abandoning mid-import', () => { expect(run.itemResults.length).toBe(3) }) + // What a caller acting on the run's leftovers depends on: `abandon()` only stops the next + // phase, so a reload issued when it is called reads the workspace while the request already + // sent is still landing. `whenIdle()` is the difference between reloading then and after. + it('whenIdle resolves only once the abandoned run has stopped writing', async () => { + const run = new ImportExecution(PLAN, deps) + let idleResolved = false + hooks.afterFirstItem = () => { + run.abandon() + void run.whenIdle().then(() => (idleResolved = true)) + // Still inside the run: the promise must not have resolved yet. + expect(run.running).toBe(true) + expect(idleResolved).toBe(false) + } + await run.run() + await run.whenIdle() + expect(run.running).toBe(false) + expect(idleResolved).toBe(true) + }) + + it('whenIdle resolves immediately when no run is in flight', async () => { + const run = new ImportExecution(PLAN, deps) + await expect(run.whenIdle()).resolves.toBeUndefined() + }) + it('stops the migrate row spinning when it is abandoned mid-migration', async () => { const run = new ImportExecution(PLAN, depsWithMigration) // After `onMigrationsStart`, which is where the row is actually set to running — diff --git a/frontend/src/lib/importWizard/execution.svelte.ts b/frontend/src/lib/importWizard/execution.svelte.ts index 6a2c39b0b6..12952525b5 100644 --- a/frontend/src/lib/importWizard/execution.svelte.ts +++ b/frontend/src/lib/importWizard/execution.svelte.ts @@ -229,6 +229,29 @@ export class ImportExecution { */ async run(): Promise { if (this.running) return + const settled = this.#runInternal() + // Handled here so an abandoned or failed run does not surface as an unhandled + // rejection through `whenIdle()`, but still reported: `#runInternal` has no `catch` of + // its own, and a throw outside its inner ones leaves a stalled run with nothing on + // screen — the console is the only place that says why. + this.#idle = settled.catch((error) => console.error('import run failed:', error)) + return settled + } + + /** + * Resolves when the run in flight at the moment of the call is no longer writing — + * immediately when there is none. Callers that act on what a run left behind need this + * rather than a poll on `running`: `abandon()` stops the run at the next phase boundary, + * so the request already sent lands after it, and reading the workspace before then reads + * it mid-write. A caller that holds the promise across the start of a *second* run is + * resolved by the first, so re-read it if the surface stays open. + */ + whenIdle(): Promise { + return this.#idle + } + #idle: Promise = Promise.resolve() + + async #runInternal(): Promise { this.#abandoned = false this.running = true runState.active = true diff --git a/frontend/src/lib/importWizard/setupStep.svelte.ts b/frontend/src/lib/importWizard/setupStep.svelte.ts new file mode 100644 index 0000000000..17f63cbd4f --- /dev/null +++ b/frontend/src/lib/importWizard/setupStep.svelte.ts @@ -0,0 +1,73 @@ +import { WorkspaceService } from '$lib/gen' +import type { ImportExecution } from './execution.svelte' + +/** + * Whether a finished import leaves a setup step behind it, and whether that is still + * being decided. + * + * Known only once the run has fetched the export and the destination's data tables can + * be compared against it, so it is false for the whole wizard until the import + * finishes — which is exactly when it is first read. `undecided` matters as much as + * `needed`: without it the run reads as finished with no fourth step, and Finish leaves + * before the check comes back and discovers a data table that is missing. + * + * Shared by the wizard route and the in-workspace modal so the two cannot disagree + * about whether an import is over. + */ +export function useSetupStep( + getExecution: () => ImportExecution | undefined, + getWorkspace: () => string | undefined +) { + let needed = $state(false) + let undecided = $state(false) + + $effect(() => { + const execution = getExecution() + const names = execution?.datatableNames ?? [] + const workspace = getWorkspace() + if (!execution?.done || !workspace) { + needed = false + undecided = false + return + } + // `resourceCount` is the referenced subset — the resources something in the project + // points at — and each one arrives as an empty stub, so any project that has them has + // something to fill in. The step itself re-checks and shows only what is genuinely + // outstanding, which is what makes a re-import quiet. + if (execution.resourceCount > 0) { + needed = true + undecided = false + return + } + if (names.length === 0) { + needed = false + undecided = false + return + } + let cancelled = false + undecided = true + void WorkspaceService.listDataTables({ workspace }) + .then((tables) => { + if (cancelled) return + const present = new Set(tables.map((t) => t.name)) + needed = names.some((n) => !present.has(n)) + }) + .catch(() => { + // Can't tell — don't invent a step the user then cannot complete. + if (!cancelled) needed = false + }) + .finally(() => { + if (!cancelled) undecided = false + }) + return () => (cancelled = true) + }) + + return { + get needed() { + return needed + }, + get undecided() { + return undecided + } + } +} diff --git a/frontend/src/lib/refreshUser.ts b/frontend/src/lib/refreshUser.ts index de20fa5d56..4c6b47e8f3 100644 --- a/frontend/src/lib/refreshUser.ts +++ b/frontend/src/lib/refreshUser.ts @@ -1,23 +1,39 @@ import { get } from 'svelte/store' -import { CancelablePromise, UserService, type GlobalUserInfo } from '$lib/gen' +import { CancelablePromise, CancelError, UserService, type GlobalUserInfo } from '$lib/gen' import { superadmin, devopsRole } from './stores.js' let promise: CancelablePromise | null = null -async function _refreshSuperadmin(): Promise { - let shouldFetch = get(superadmin) == undefined || get(devopsRole) == undefined +/** + * `force` asks the server even when the stores already hold an answer. Worth it where a wrong + * answer changes what the page offers rather than how it looks: a logged-out load sets both + * stores to `false` — the request 401s — and without `force` nothing asks again for the rest + * of the session, so the user who signs in next reads as neither superadmin nor devops. + */ +async function _refreshSuperadmin(opts?: { force?: boolean }): Promise { + let shouldFetch = opts?.force || get(superadmin) == undefined || get(devopsRole) == undefined if (!shouldFetch) return undefined promise?.cancel() - promise = UserService.globalWhoami() + // Held locally so the check at the end can tell this request from a later caller's, which + // by then owns `promise`. + const mine = UserService.globalWhoami() + promise = mine try { - const me = await promise + const me = await mine superadmin.set(me.super_admin ? me.email : false) devopsRole.set(me.devops || me.super_admin ? me.email : false) } catch (error) { - superadmin.set(false) - devopsRole.set(false) - console.error('error refreshing superadmin/devops role', error) + // A cancellation says nothing about this user, so it must not be written down as an + // answer: `clearStores` cancels on logout, and a second caller cancels the first — and + // `false` here is precisely the stale state `force` exists to get out of. + if (!(error instanceof CancelError)) { + superadmin.set(false) + devopsRole.set(false) + console.error('error refreshing superadmin/devops role', error) + } } - promise = null + // Only if nobody has started another: clearing a live request's handle would put it beyond + // the reach of `cancel()`, and it would then land on a session that had been cleared. + if (promise === mine) promise = null } export const refreshSuperadmin = Object.assign(_refreshSuperadmin, { diff --git a/frontend/src/lib/stores.ts b/frontend/src/lib/stores.ts index ed5bc6f025..229ecff7ad 100644 --- a/frontend/src/lib/stores.ts +++ b/frontend/src/lib/stores.ts @@ -68,8 +68,6 @@ export function clearWorkspaceFromStorage() { sessionStorage.removeItem('workspace') } -export const tutorialsToDo = writable([]) -export const skippedAll = writable(false) export const globalEmailInvite = writable('') export const awarenessStore = writable>(undefined) export const enterpriseLicense = writable(undefined) @@ -120,6 +118,10 @@ export const superadmin = writable(undefined) export const devopsRole = writable(undefined) export const lspTokenStore = writable(undefined) export const hubBaseUrlStore = writable(DEFAULT_HUB_BASE_URL) +// Whether the store above is the instance's answer or still the default it was seeded with. +// It reads as the public hub either way, which is fine for a link and wrong for anything +// deciding what may be reported about a hub — those must treat unknown as private. +export const hubBaseUrlKnown = writable(false) export const wsBaseUrlStore = writable(undefined) export const disableHubStore = writable(false) // What a superadmin standing in a workspace they are not a member of needs to see it as a @@ -333,8 +335,6 @@ export const workspaceColor: Readable = derived( } ) -export const isCurrentlyInTutorial: StateStore = createState({ val: false }) - export function getFlatTableNamesFromSchema(dbSchema: DBSchema | undefined): string[] { const schema = dbSchema?.schema ?? {} const tableNames: string[] = [] diff --git a/frontend/src/lib/tutorialUtils.ts b/frontend/src/lib/tutorialUtils.ts deleted file mode 100644 index 02222138e8..0000000000 --- a/frontend/src/lib/tutorialUtils.ts +++ /dev/null @@ -1,224 +0,0 @@ -import { get } from 'svelte/store' -import { tutorialsToDo, skippedAll } from './stores' -import { UserService } from './gen' -import { TUTORIALS_CONFIG } from './tutorials/config' - -/** - * LocalStorage key for tracking if the tutorial banner has been dismissed. - * Shared between tutorialUtils and TutorialBanner component. - */ -export const TUTORIAL_BANNER_DISMISSED_KEY = 'tutorial_banner_dismissed' - -/** - * Get the maximum tutorial index from the config. - * This ensures we don't hardcode the max ID and it automatically updates when tutorials are added. - */ -function getMaxTutorialId(): number { - let maxId = 0 - for (const tab of Object.values(TUTORIALS_CONFIG)) { - for (const tutorial of tab.tutorials) { - if (tutorial.index !== undefined && tutorial.index > maxId) { - maxId = tutorial.index - } - } - } - return maxId -} - -const MAX_TUTORIAL_ID = getMaxTutorialId() - -/** - * Helper function to calculate tutorial progress for a given set of tutorial indexes. - * Returns total count. For completed count, use in component with reactive store access. - */ -export function getTutorialProgressTotal(tutorialIndexes: Record): number { - return Object.values(tutorialIndexes).length -} - -/** - * Helper function to calculate completed tutorials count. - * Must be called with current tutorialsToDo array. - */ -export function getTutorialProgressCompleted( - tutorialIndexes: Record, - tutorialsToDoArray: number[] -): number { - return Object.values(tutorialIndexes).filter((index) => !tutorialsToDoArray.includes(index)) - .length -} - -export async function updateProgress(id: number) { - const bef = get(tutorialsToDo) - const aft = bef.filter((x) => x != id) - tutorialsToDo.set(aft) - skippedAll.set(false) // Mark as not skipped when completing a tutorial - let bits = 0 - for (let i = 0; i <= MAX_TUTORIAL_ID; i++) { - let mask = 1 << i - if (!aft.includes(i)) { - bits = bits | mask - } - } - await UserService.updateTutorialProgress({ requestBody: { progress: bits, skipped_all: false } }) -} - -export async function skipAllTodos() { - let bits = 0 - for (let i = 0; i <= MAX_TUTORIAL_ID; i++) { - let mask = 1 << i - bits = bits | mask - } - tutorialsToDo.set([]) - skippedAll.set(true) - - await UserService.updateTutorialProgress({ requestBody: { progress: bits, skipped_all: true } }) -} - -export async function resetAllTodos() { - let todos: number[] = [] - for (let i = 0; i <= MAX_TUTORIAL_ID; i++) { - todos.push(i) - } - tutorialsToDo.set(todos) - skippedAll.set(false) - - await UserService.updateTutorialProgress({ requestBody: { progress: 0, skipped_all: false } }) -} - -/** - * Skip (mark as complete) all tutorials in a specific set of indexes - */ -export async function skipTutorialsByIndexes(tutorialIndexes: number[]) { - const currentTodos = get(tutorialsToDo) - const aft = currentTodos.filter((x) => !tutorialIndexes.includes(x)) - tutorialsToDo.set(aft) - - // Get current progress bits - const currentResponse = await UserService.getTutorialProgress() - let bits: number = currentResponse.progress ?? 0 - - // Set bits for the specified indexes - for (const index of tutorialIndexes) { - const mask = 1 << index - bits = bits | mask - } - - // Only set skipped_all to true if ALL tutorials are now complete - const allComplete = aft.length === 0 - await UserService.updateTutorialProgress({ - requestBody: { - progress: bits, - skipped_all: allComplete - } - }) -} - -/** - * Reset (mark as incomplete) all tutorials in a specific set of indexes - */ -export async function resetTutorialsByIndexes(tutorialIndexes: number[]) { - const currentTodos = get(tutorialsToDo) - const aft = [...new Set([...currentTodos, ...tutorialIndexes])] - tutorialsToDo.set(aft) - skippedAll.set(false) - - // Get current progress bits - const currentResponse = await UserService.getTutorialProgress() - let bits: number = currentResponse.progress ?? 0 - - // Clear bits for the specified indexes - for (const index of tutorialIndexes) { - const mask = 1 << index - bits = bits & ~mask - } - - await UserService.updateTutorialProgress({ - requestBody: { - progress: bits, - skipped_all: false - } - }) -} - -/** - * Update a single tutorial's completion status by index - */ -async function updateTutorialStatusByIndex(tutorialIndex: number, completed: boolean) { - const currentTodos = get(tutorialsToDo) - const isInTodos = currentTodos.includes(tutorialIndex) - - // Only update if the status needs to change - // isInTodos = true means NOT completed, isInTodos = false means completed - // So if completed === !isInTodos, we're already in the desired state - if (completed === !isInTodos) { - return // Already in the desired state - } - - // Update todos list - const aft = completed - ? currentTodos.filter((x) => x !== tutorialIndex) - : [...currentTodos, tutorialIndex] - tutorialsToDo.set(aft) - skippedAll.set(false) - - // Get current progress bits - const currentResponse = await UserService.getTutorialProgress() - let bits: number = currentResponse.progress ?? 0 - - // Update bit for this tutorial index - const mask = 1 << tutorialIndex - bits = completed ? bits | mask : bits & ~mask - - await UserService.updateTutorialProgress({ - requestBody: { - progress: bits, - skipped_all: false - } - }) -} - -/** - * Reset (mark as incomplete) a single tutorial by index - */ -export async function resetTutorialByIndex(tutorialIndex: number) { - await updateTutorialStatusByIndex(tutorialIndex, false) -} - -/** - * Mark a single tutorial as completed by index - */ -export async function completeTutorialByIndex(tutorialIndex: number) { - await updateTutorialStatusByIndex(tutorialIndex, true) -} - -export async function syncTutorialsTodos() { - const response = await UserService.getTutorialProgress() - const bits: number = response.progress! - const skipped: boolean = response.skipped_all ?? false - const todos: number[] = [] - for (let i = 0; i <= MAX_TUTORIAL_ID; i++) { - let mask = 1 << i - if ((bits & mask) == 0) { - todos.push(i) - } - } - tutorialsToDo.set(todos) - skippedAll.set(skipped) -} - -export function tutorialInProgress() { - const svg = document.getElementsByClassName('driver-overlay driver-overlay-animated') - - return svg.length > 0 -} - -/** - * Check if tutorials should be hidden from the main menu. - * Returns true if all tutorials are completed OR user skipped all. - */ -export function shouldHideTutorialsFromMainMenu(): boolean { - const todos = get(tutorialsToDo) - const skipped = get(skippedAll) - // Hide if all tutorials are completed OR user skipped all - return todos.length === 0 || skipped -} diff --git a/frontend/src/lib/tutorials/config.ts b/frontend/src/lib/tutorials/config.ts deleted file mode 100644 index 082dc0473d..0000000000 --- a/frontend/src/lib/tutorials/config.ts +++ /dev/null @@ -1,159 +0,0 @@ -import type { ComponentType } from 'svelte' -import { Workflow, GraduationCap, Wrench, PlayCircle, Link2, History } from 'lucide-svelte' -import { base } from '$lib/base' -import type { Role } from './roleUtils' - -export interface TutorialConfig { - id: string - icon: ComponentType - title: string - description: string - onClick: () => void - index?: number // Bitmask index in the database (for progress tracking) - active?: boolean // Whether this tutorial is active and should be displayed (default: true) - comingSoon?: boolean - roles?: Role[] // Roles that can access this tutorial (if not specified, available to everyone) - order?: number -} - -export interface TabConfig { - label: string - tutorials: TutorialConfig[] - roles?: Role[] // Roles that can access this tab category (if not specified, available to everyone) - progressBar?: boolean // Whether to display the progress bar for this tab (default: true) - active?: boolean // Whether this tab category is active and should be displayed (default: true) -} - -export type TabId = 'quickstart' | 'app_editor' - -/** - * Get tutorial index from config by tutorial ID. - * Throws an error if the tutorial or its index is not found. - */ -export function getTutorialIndex(id: string): number { - for (const tab of Object.values(TUTORIALS_CONFIG)) { - const tutorial = tab.tutorials.find((t) => t.id === id) - if (tutorial?.index !== undefined) return tutorial.index - } - throw new Error(`Tutorial index not found for id: ${id}. Make sure the tutorial has an index defined in config.`) -} - -// Available roles : developer, admin, operator - -export const TUTORIALS_CONFIG: Record = { - quickstart: { - label: 'Quickstart', - roles: ['admin', 'developer', 'operator'], - progressBar: true, - active: true, - tutorials: [ - { - id: 'workspace-onboarding', - icon: GraduationCap, - title: 'Workspace onboarding', - description: 'Discover the basics of Windmill with a quick tour of the workspace.', - onClick: () => { - window.location.href = `${base}/?tutorial=workspace-onboarding` - }, - index: 1, - active: true, - comingSoon: false, - roles: ['developer', 'admin'], - order: 1 - }, - { - id: 'flow-live-tutorial', - icon: Workflow, - title: 'Build a flow', - description: 'Learn how to build workflows in Windmill with our interactive tutorial.', - onClick: () => { - window.location.href = `${base}/flows/add?tutorial=flow-live-tutorial` - }, - index: 2, - active: true, - comingSoon: false, - roles: ['developer', 'admin'], - order: 2 - }, - { - id: 'troubleshoot-flow', - icon: Wrench, - title: 'Fix a broken flow', - description: 'Learn how to monitor and debug your script and flow executions.', - onClick: () => { - window.location.href = `${base}/flows/add?tutorial=troubleshoot-flow` - }, - index: 3, - active: true, - comingSoon: false, - roles: ['admin','developer'], - order: 3 - }, - { - id: 'runs-tutorial', - icon: History, - title: 'Discover your monitoring dashboard', - description: 'Learn how to monitor, filter, and manage your script and flow executions.', - onClick: () => { - window.location.href = `${base}/runs?tutorial=runs-tutorial` - }, - index: 7, - active: true, - comingSoon: false, - roles: ['admin', 'developer','operator'], - order: 4 - }, - { - id: 'workspace-onboarding-operator', - icon: GraduationCap, - title: 'Workspace onboarding', - description: 'Discover the basics of Windmill with a quick tour of the workspace.', - onClick: () => { - window.location.href = `${base}/?tutorial=workspace-onboarding-operator` - }, - index: 6, - active: true, - comingSoon: false, - roles: ['operator'], - order: 1 - }, - ] - }, - app_editor: { - label: 'App Editor', - roles: ['developer', 'admin'], - progressBar: false, - active: true, - tutorials: [ - { - id: 'backgroundrunnables', - icon: PlayCircle, - title: 'Background runnables', - description: 'Learn how to create and use background runnables in your apps.', - onClick: () => { - window.location.href = `${base}/apps/add?tutorial=backgroundrunnables` - }, - index: 4, - active: true, - comingSoon: false, - roles: ['developer','admin'], - order: 4 - }, - { - id: 'connection', - icon: Link2, - title: 'Connection', - description: 'Learn how to connect component inputs to outputs in your apps.', - onClick: () => { - window.location.href = `${base}/apps/add?tutorial=connection` - }, - index: 5, - active: true, - comingSoon: false, - roles: ['developer', 'admin'], - order: 5 - } - ] - } -} as const - diff --git a/frontend/src/lib/tutorials/roleUtils.ts b/frontend/src/lib/tutorials/roleUtils.ts deleted file mode 100644 index a727fca8d3..0000000000 --- a/frontend/src/lib/tutorials/roleUtils.ts +++ /dev/null @@ -1,68 +0,0 @@ -import type { UserExt } from '$lib/stores' - -export type Role = 'admin' | 'developer' | 'operator' - -/** - * Get the effective role of a user based on their database flags. - * - Admin: user.is_admin === true - * - Operator: user.operator === true (and not admin) - * - Developer: default (neither admin nor operator) - */ -export function getUserEffectiveRole(user: UserExt | null | undefined): Role | null { - if (!user) return null - if (user.is_admin) return 'admin' - if (user.operator) return 'operator' - return 'developer' -} - -/** - * Check if a role has access to a required role. - * This is the core role-checking logic used by both normal and preview modes. - */ -function checkRoleMatch( - userRole: Role, - requiredRole: Role -): boolean { - if (requiredRole === 'admin') return userRole === 'admin' - if (requiredRole === 'operator') return userRole === 'operator' || userRole === 'admin' - if (requiredRole === 'developer') return userRole === 'developer' || userRole === 'admin' - return false -} - -/** - * Check if a user or preview role has access based on a roles array. - * This is the unified function that handles both normal user access and admin preview mode. - */ -export function hasRoleAccess( - user: UserExt | null | undefined, - roles?: Role[], - previewRole?: Role -): boolean { - // No roles specified = available to everyone - if (!roles || roles.length === 0) return true - - // If previewRole is provided, use it (admin preview mode) - // Otherwise, derive role from user - const effectiveRole = previewRole ?? getUserEffectiveRole(user) - if (!effectiveRole) return false - - // Check if effective role has any of the required roles - return roles.some((role) => checkRoleMatch(effectiveRole, role)) -} - -/** - * Check if a preview role has access based on a roles array. - * Used by admins to preview what other roles can see. - * Uses exact role matching - only shows tutorials explicitly marked for the preview role. - */ -export function hasRoleAccessForPreview( - previewRole: Role, - roles?: Role[] -): boolean { - // No roles specified = available to everyone - if (!roles || roles.length === 0) return true - - // Exact role match - tutorial must explicitly include the preview role - return roles.includes(previewRole) -} - diff --git a/frontend/src/lib/utils/featureUsage.test.ts b/frontend/src/lib/utils/featureUsage.test.ts index ef60cd2b36..1857efb0c0 100644 --- a/frontend/src/lib/utils/featureUsage.test.ts +++ b/frontend/src/lib/utils/featureUsage.test.ts @@ -1,10 +1,34 @@ -import { describe, expect, it, vi } from 'vitest' +import { beforeEach, describe, expect, it, vi } from 'vitest' vi.mock('$lib/gen', () => ({ OpenAPI: { BASE: '/api' } })) -vi.mock('$lib/stores', () => ({ workspaceStore: { subscribe: () => () => {} } })) + +// Stores `get()` can read, so a test can say which hub the instance points at and whether +// the instance has answered at all. +const hubBaseUrl = vi.hoisted(() => { + const readable = (initial: T) => { + let value = initial + return { + set: (v: T) => (value = v), + store: { + subscribe: (run: (v: T) => void) => { + run(value) + return () => {} + } + } + } + } + return { url: readable('https://hub.windmill.dev'), known: readable(true) } +}) + +vi.mock('$lib/stores', () => ({ + workspaceStore: { subscribe: () => () => {} }, + hubBaseUrlStore: hubBaseUrl.url.store, + hubBaseUrlKnown: hubBaseUrl.known.store +})) import { createFeatureUsageBuffer, + hubProjectUsageKey, hubScriptUsageKey, type FeatureUsageEventPayload } from './featureUsage' @@ -107,3 +131,50 @@ describe('hubScriptUsageKey', () => { ).toBe('acme/list_a_user_s_items_sorted') }) }) + +describe('hubProjectUsageKey', () => { + // The fixture is module-level and mutable, so each case states the world it needs rather + // than inheriting whatever the case above it left behind. + beforeEach(() => { + hubBaseUrl.url.set('https://hub.windmill.dev') + hubBaseUrl.known.set(true) + }) + + it('reports the slug for every spelling of the public hub', () => { + for (const hub of [ + 'https://hub.windmill.dev', + 'http://hub.windmill.dev/', + 'HTTPS://hub.windmill.dev', + 'https://HUB.WINDMILL.DEV', + 'https://hub.windmill.dev:443', + ' https://hub.windmill.dev ' + ]) { + hubBaseUrl.url.set(hub) + expect(hubProjectUsageKey('stripe-invoices'), hub).toBe('stripe-invoices') + } + }) + + it('answers private until the instance has said which hub it points at', () => { + // The store is seeded with the public hub, so a settings read that failed must not + // read as permission to report the name. + hubBaseUrl.known.set(false) + expect(hubProjectUsageKey('acme-payroll')).toBe('private') + hubBaseUrl.known.set(true) + expect(hubProjectUsageKey('acme-payroll')).toBe('acme-payroll') + }) + + it("keeps a private hub's project names off the wire", () => { + // The slug is the customer's own content on an instance running its own hub, and the + // disclosure only claims public project names. + for (const hub of [ + 'https://hub.internal.example', + 'https://hub.windmill.dev.evil.example', + 'https://windmill.dev', + 'hub.windmill.dev', + 'not a url' + ]) { + hubBaseUrl.url.set(hub) + expect(hubProjectUsageKey('acme-payroll'), hub).toBe('private') + } + }) +}) diff --git a/frontend/src/lib/utils/featureUsage.ts b/frontend/src/lib/utils/featureUsage.ts index 793d137593..31b33153cf 100644 --- a/frontend/src/lib/utils/featureUsage.ts +++ b/frontend/src/lib/utils/featureUsage.ts @@ -1,7 +1,7 @@ import { get } from 'svelte/store' import { OpenAPI } from '$lib/gen' -import { workspaceStore } from '$lib/stores' -import { PRIVATE_HUB_MIN_VERSION } from '$lib/hub' +import { hubBaseUrlKnown, hubBaseUrlStore, workspaceStore } from '$lib/stores' +import { DEFAULT_HUB_BASE_URL, PRIVATE_HUB_MIN_VERSION } from '$lib/hub' // Anonymous product-usage counters (e.g. AI session activity), batched into the // backend `feature_usage` accumulator. Only aggregated counts ever leave the @@ -187,3 +187,34 @@ export function hubScriptUsageKey(script: { if (!app) return PRIVATE_HUB_KEY return (summary ? `${app}/${summary}` : app).slice(0, 100) } + +/** + * A hub project's slug is only reportable when it names something on the public hub. An + * instance pointed at its own hub imports its own projects, whose names are the customer's + * content — the same reason `hubScriptUsageKey` collapses a private script to `private`, + * and what the disclosure means by "the name of any public hub project". + * + * Compared by host, so the port, scheme and trailing slash an operator may have typed do + * not decide it. Anything unparseable, and anything not yet read, answers private. + */ +export function hubProjectUsageKey(slug: string): string { + // `hubBaseUrlKnown` and not the URL alone: the store is seeded with the public hub, so an + // instance whose setting could not be read would otherwise report its own project names. + if (!get(hubBaseUrlKnown) || !isPublicHub(get(hubBaseUrlStore))) return PRIVATE_HUB_KEY + return slug.slice(0, 100) +} + +function isPublicHub(hub: string): boolean { + const host = (url: string): string | undefined => { + try { + const parsed = new URL(url.trim()) + return parsed.protocol === 'http:' || parsed.protocol === 'https:' + ? parsed.hostname.replace(/\.$/, '').toLowerCase() + : undefined + } catch { + return undefined + } + } + const configured = host(hub) + return configured !== undefined && configured === host(DEFAULT_HUB_BASE_URL) +} diff --git a/frontend/src/lib/workspaceCreation.test.ts b/frontend/src/lib/workspaceCreation.test.ts new file mode 100644 index 0000000000..975d1599b5 --- /dev/null +++ b/frontend/src/lib/workspaceCreation.test.ts @@ -0,0 +1,88 @@ +import { describe, expect, it, vi } from 'vitest' + +// The module reaches the API for the username policy and the workspace list; the name +// helper touches neither. `getGlobal` is a spy so the policy's failure path can be driven. +const getGlobal = vi.fn() +vi.mock('./gen', () => ({ + SettingService: { + get getGlobal() { + return getGlobal + } + }, + UserService: {}, + WorkspaceService: {} +})) +vi.mock('./stores', () => ({ usersWorkspaceStore: { set: () => {} } })) +vi.mock('./storeUtils', () => ({ switchWorkspace: () => {} })) +vi.mock('./cloud', () => ({ isCloudHosted: () => false })) + +import { defaultWorkspaceName, loadUsernamePolicy, usernameFromName } from './workspaceCreation' + +describe('defaultWorkspaceName', () => { + it('names the workspace after the person, not the address', () => { + expect(defaultWorkspaceName(undefined, 'bob@example.com')).toBe("Bob's workspace") + expect(defaultWorkspaceName(undefined, 'ada.lovelace@example.com')).toBe( + "Ada Lovelace's workspace" + ) + expect(defaultWorkspaceName(undefined, 'jean-luc_picard+wm@example.com')).toBe( + "Jean Luc Picard Wm's workspace" + ) + }) + + it('prefers the name the login provider gave', () => { + expect(defaultWorkspaceName('Ruben', 'r.k@example.com')).toBe("Ruben's workspace") + // Blank is not a name: fall back rather than produce "'s workspace". + expect(defaultWorkspaceName(' ', 'bob@example.com')).toBe("Bob's workspace") + }) + + it('falls back rather than offering a name the backend refuses', () => { + // Over the 50-char cap the field would be prefilled with something rejected on submit. + expect(defaultWorkspaceName('Bartholomew Maximilian Featherstonehaugh III', undefined)).toBe( + 'My workspace' + ) + // Nothing to derive from at all. + expect(defaultWorkspaceName(undefined, undefined)).toBe('My workspace') + expect(defaultWorkspaceName(undefined, '@example.com')).toBe('My workspace') + }) +}) + +describe('usernameFromName', () => { + // The `proper_username` constraint is `^[\w-]+$`, so a suggestion outside it is posted and + // then refused by the database, with the form showing nothing that explains why. + it('keeps only what the username constraint accepts', () => { + expect(usernameFromName("O'Connor")).toBe('oconnor') + expect(usernameFromName('alice+demo')).toBe('alicedemo') + expect(usernameFromName('Jean-Luc')).toBe('jean-luc') + expect(usernameFromName('ada.lovelace')).toBe('adalovelace') + }) + + it('answers undefined when nothing usable is left', () => { + // The caller opens the full form instead of prefilling something unusable. + expect(usernameFromName('++')).toBeUndefined() + expect(usernameFromName('')).toBeUndefined() + }) + + it('answers undefined rather than a value the column cannot hold', () => { + // `usr.username` is VARCHAR(50) while the name and email it is derived from run to 255, + // and `create_workspace` inserts it untruncated. + expect(usernameFromName('a'.repeat(50))).toBe('a'.repeat(50)) + expect(usernameFromName('a'.repeat(51))).toBeUndefined() + }) +}) + +describe('loadUsernamePolicy', () => { + // Neither default is safe — `create_workspace` refuses a username on an automating + // instance and requires one otherwise — so an unreadable setting has to reach the caller + // as a failure rather than as a guess it cannot tell apart from an answer. + it('rejects rather than guessing when the setting cannot be read', async () => { + getGlobal.mockRejectedValueOnce(new Error('502')) + await expect(loadUsernamePolicy()).rejects.toThrow('502') + }) + + it('automates when the setting says so, and when it is unset', async () => { + getGlobal.mockResolvedValueOnce(true) + expect(await loadUsernamePolicy()).toEqual({ automate: true }) + getGlobal.mockResolvedValueOnce(null) + expect(await loadUsernamePolicy()).toEqual({ automate: true }) + }) +}) diff --git a/frontend/src/lib/workspaceCreation.ts b/frontend/src/lib/workspaceCreation.ts index 49e4bdbea3..e4a9045f42 100644 --- a/frontend/src/lib/workspaceCreation.ts +++ b/frontend/src/lib/workspaceCreation.ts @@ -14,6 +14,7 @@ import { usersWorkspaceStore } from '$lib/stores' import { switchWorkspace } from '$lib/storeUtils' import { isCloudHosted } from '$lib/cloud' import { base } from '$lib/base' +import { WORKSPACE_NAME_MAX_LENGTH } from '$lib/utils/workspaceId' /** * Whether this user may create a workspace at all. Self-hosted instances default @@ -44,11 +45,34 @@ export interface UsernamePolicy { suggested?: string } +/** What `usr.username` holds, and neither the provider name nor the email is bounded by it. */ +const USERNAME_MAX_LENGTH = 50 + +/** + * A username the whole `usr.username` contract accepts: the `proper_username` constraint + * (`^[\w-]+$`, so word characters and hyphens and nothing else) and the column's own 50 + * characters. Anything outside the class is dropped rather than substituted — `O'Connor` is + * `oconnor`, not `o-connor`. + * + * Undefined where nothing usable is left or where what is left is too long, which is the + * caller's cue to ask for one: `create_workspace` inserts this value with no truncation, so a + * name the column refuses would fail on insert with nothing on screen naming the field. + */ +export function usernameFromName(name: string): string | undefined { + const cleaned = name.toLowerCase().replace(/[^\w-]/g, '') + return cleaned === '' || cleaned.length > USERNAME_MAX_LENGTH ? undefined : cleaned +} + /** * `createWorkspace` rejects a username when the instance automates them and * requires one when it does not, so the field only exists in the second case. */ export async function loadUsernamePolicy(): Promise { + // Rejects rather than defaulting when the setting cannot be read, because neither + // default is safe: `create_workspace` refuses a username on an instance that automates + // them and requires one on an instance that does not (`workspaces.rs:5820`). A caller + // that cannot read this cannot pick a request shape, and must say so instead of posting + // one of the two the server rejects. const automate = ((await SettingService.getGlobal({ key: 'automate_username_creation' @@ -57,7 +81,7 @@ export async function loadUsernamePolicy(): Promise { try { const me = await UserService.globalWhoami() const from = me.name ? me.name.split(' ')[0] : me.email.split('@')[0] - return { automate: false, suggested: from.replace(/\./g, '').toLowerCase() } + return { automate: false, suggested: usernameFromName(from) } } catch { return { automate: false } } @@ -78,3 +102,28 @@ export async function enterNewWorkspace(id: string): Promise { await refreshWorkspaceList() switchWorkspace(id) } + +/** + * How long a screen that hands over to a workspace stays up, whatever the server does. + * Creating or naming one takes a few hundred milliseconds, and a button that swaps the page in + * that time reads as nothing having happened — the floor is what makes it read as an action + * that ran, and it covers the workspace layout's first load on the other side. + */ +export const WORKSPACE_HANDOVER_MS = 900 + +/** + * What to call a workspace before its owner has said. The login provider's name when it gave + * one, else the email local part read as a name: `bob@…` is Bob, `ada.lovelace@…` is Ada + * Lovelace. Capped at what `create_workspace` accepts, since it is prefilled rather than + * typed and a name the server would reject must never appear in the field. + */ +export function defaultWorkspaceName(name: string | undefined, email: string | undefined): string { + const display = (name?.trim() || (email ?? '').split('@')[0]) + .split(/[._\-+\s]+/) + .filter(Boolean) + .map((word) => word.charAt(0).toUpperCase() + word.slice(1)) + .join(' ') + .trim() + const proposed = display ? `${display}'s workspace` : 'My workspace' + return proposed.length > WORKSPACE_NAME_MAX_LENGTH ? 'My workspace' : proposed +} diff --git a/frontend/src/routes/(root)/(logged)/+layout.svelte b/frontend/src/routes/(root)/(logged)/+layout.svelte index e82d328c76..5aa90b80a2 100644 --- a/frontend/src/routes/(root)/(logged)/+layout.svelte +++ b/frontend/src/routes/(root)/(logged)/+layout.svelte @@ -32,6 +32,7 @@ type UserExt, defaultScripts, hubBaseUrlStore, + hubBaseUrlKnown, wsBaseUrlStore, disableHubStore, usedTriggerKinds, @@ -60,7 +61,6 @@ } from '$lib/components/sidebar/FavoriteMenu.svelte' import { SUPERADMIN_SETTINGS_HASH, USER_SETTINGS_HASH } from '$lib/components/sidebar/settings' import { isCloudHosted } from '$lib/cloud' - import { syncTutorialsTodos } from '$lib/tutorialUtils' import { PanelLeftClose, PanelLeftOpen, Home, Play, Search, WandSparkles } from 'lucide-svelte' import { getUserExt } from '$lib/user' import { confirmPendingLoginMethod } from '$lib/lastLoginMethod' @@ -468,7 +468,6 @@ function onLoad() { loadFavorites() - syncTutorialsTodos() loadHubBaseUrl() loadWsBaseUrl() loadDisableHub() @@ -476,10 +475,18 @@ } async function loadHubBaseUrl() { - $hubBaseUrlStore = - ((await SettingService.getGlobal({ key: 'hub_accessible_url' })) as string) || - ((await SettingService.getGlobal({ key: 'hub_base_url' })) as string) || - DEFAULT_HUB_BASE_URL + // A read that throws leaves the store on its seeded default, which names the public hub + // — so the flag, not the value, is what says the instance has answered. An instance that + // simply has no setting still answers: the chain falls through to the default. + try { + $hubBaseUrlStore = + ((await SettingService.getGlobal({ key: 'hub_accessible_url' })) as string) || + ((await SettingService.getGlobal({ key: 'hub_base_url' })) as string) || + DEFAULT_HUB_BASE_URL + $hubBaseUrlKnown = true + } catch (error) { + console.error('Could not read the hub URL:', error) + } } async function loadWsBaseUrl() { @@ -1107,7 +1114,7 @@
    + Tag + Waiting + Next job's wait + Running + Workers + + + + {#each rows as s (s.tag)} + + {s.tag} + {s.waiting} + + {s.delay === undefined ? '-' : msToReadableTime(s.delay * 1000, 0)} + + {s.running} + + {#if unserved(s)} + + + No worker currently pulls this tag + + {:else} + {s.workers} + {/if} + + + {/each} + + + {/if} + From fa73539839071491fb2cbad9242f52ad22b975bf Mon Sep 17 00:00:00 2001 From: AlexRV12 <71396855+AlexRV12@users.noreply.github.com> Date: Thu, 10 Sep 2026 20:41:21 +0200 Subject: [PATCH 064/144] fix(ai-chat): test_run_flow could test a different flow than the one asked (#11066) * fix(ai-chat): test_run_flow could test a different flow than the one asked Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01TmQDoFXVPwYyEN8PfV2oL7 * fix(ai-chat): prefer the flow editor stored at the path over one renamed to it Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01TmQDoFXVPwYyEN8PfV2oL7 * test(ai-chat): default the flow helpers factory and trim duplicated setup Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01TmQDoFXVPwYyEN8PfV2oL7 * refactor(ai-chat): resolve the flow editor to run by its storage path Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01TmQDoFXVPwYyEN8PfV2oL7 * refactor(ai-chat): move the editor storage path context out of sessions Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01TmQDoFXVPwYyEN8PfV2oL7 --------- Co-authored-by: Claude Opus 5 (1M context) --- .../src/lib/components/FlowBuilder.svelte | 7 +++ .../copilot/chat/AIChatManager.svelte.ts | 24 ++++++- .../copilot/chat/AIChatManager.test.ts | 48 +++++++++----- .../copilot/chat/flow/FlowAIChat.svelte | 5 ++ .../lib/components/copilot/chat/flow/core.ts | 4 ++ .../copilot/chat/global/core.test.ts | 62 ++++++++++++++++--- .../components/copilot/chat/global/core.ts | 24 ++++--- .../components/editorStoragePathContext.ts | 19 ++++++ .../sessions/SessionEditorTarget.svelte | 5 ++ 9 files changed, 167 insertions(+), 31 deletions(-) create mode 100644 frontend/src/lib/components/editorStoragePathContext.ts diff --git a/frontend/src/lib/components/FlowBuilder.svelte b/frontend/src/lib/components/FlowBuilder.svelte index 2ea94f3606..6888c61f2d 100644 --- a/frontend/src/lib/components/FlowBuilder.svelte +++ b/frontend/src/lib/components/FlowBuilder.svelte @@ -110,6 +110,7 @@ import { isCloudHosted } from '$lib/cloud' import { UserDraft } from '$lib/userDraft.svelte' import { setOpenInSessionHandoff } from './sessions/openInSessionContext' + import { getEditorStoragePath, setEditorStoragePath } from './editorStoragePathContext' let { initialPath = $bindable(''), @@ -819,6 +820,12 @@ return entries } + // The storage path this editor is bound to, narrowing whatever an outer mount + // published (a session tab): the full-page editor holds its own, a drawer mount + // holds none. + const outerStoragePath = getEditorStoragePath() + setEditorStoragePath(() => liveEditorDraftStoragePath ?? outerStoragePath?.()) + // "Open in AI session" target: the URL draft path the editor loads/saves by // (which for a new flow differs from the live-edited friendly `$pathStore`), // falling back to `$pathStore` in drawer mounts that carry no storage path. diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts index 371379939f..aafb6a2ed1 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts @@ -705,7 +705,13 @@ export class AIChatManager { >(undefined) scriptEditorShowDiffMode = $state<(() => void) | undefined>(undefined) scriptEditorGetLintErrors = $state<(() => ScriptLintResult) | undefined>(undefined) + /** The editor a FLOW-mode chat belongs to: the page owning the chat names itself here, and a + * nested editor (a subflow drawer) takes it over while it is open. Unset in a session chat, + * which keeps every open editor tab mounted and could only name an arbitrary one — a session + * resolves an editor by its storage path through `flowEditorFor`. */ flowAiChatHelpers = $state(undefined) + /** Every mounted flow editor. */ + #flowEditors = new Set() appAiChatHelpers = $state(undefined) /** Datatable creation policy: enabled flag, datatable name, and optional schema */ datatableCreationPolicy = $state<{ @@ -2415,7 +2421,8 @@ export class AIChatManager { openArtifact: this.openArtifact } : {}), - testActiveFlow: async (args?: Record) => this.flowAiChatHelpers?.testFlow(args), + testActiveFlow: async (storagePath: string, args?: Record) => + this.flowEditorFor(storagePath)?.testFlow(args), getModifiedItems: () => (this.modifiedItems ? [...this.modifiedItems] : undefined), attachedFiles: this.attachedFiles, getUserInstructions: () => getUserCustomPrompts()[AIMode.GLOBAL] ?? '', @@ -4664,7 +4671,11 @@ export class AIChatManager { } setFlowHelpers = (flowHelpers: FlowAIChatHelpers) => { - this.flowAiChatHelpers = flowHelpers + this.#flowEditors.add(flowHelpers) + // Only a chat that can reach FLOW mode names an editor (see `flowAiChatHelpers`). + if (!this.isSessionChat) { + this.flowAiChatHelpers = flowHelpers + } untrack(() => { if (this.autoAcceptEditsActive) { this.acceptPendingFlowEdits(flowHelpers) @@ -4672,10 +4683,17 @@ export class AIChatManager { }) return () => { - this.flowAiChatHelpers = undefined + this.#flowEditors.delete(flowHelpers) + if (!this.isSessionChat) { + this.flowAiChatHelpers = undefined + } } } + private flowEditorFor(storagePath: string): FlowAIChatHelpers | undefined { + return [...this.#flowEditors].find((helpers) => helpers.getStoragePath() === storagePath) + } + // Registered by the /pipeline editor while it is mounted. Rebuilds the global // tool set so the pipeline tools appear (and disappear on unregister). Pipeline // AI edits apply directly as drafts, so there is nothing to auto-accept. diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts index 54856a016d..c0bc289002 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts @@ -191,15 +191,18 @@ beforeEach(() => { }) function createFlowHelpers({ - hasPendingChanges, - acceptAllModuleActions, - testFlow = vi.fn() + hasPendingChanges = () => false, + acceptAllModuleActions = vi.fn(), + testFlow = vi.fn(), + storagePath = 'u/admin/live_flow' }: { - hasPendingChanges: () => boolean - acceptAllModuleActions: () => void + hasPendingChanges?: () => boolean + acceptAllModuleActions?: () => void testFlow?: FlowAIChatHelpers['testFlow'] -}): FlowAIChatHelpers { + storagePath?: string +} = {}): FlowAIChatHelpers { return { + getStoragePath: () => storagePath, getFlowAndSelectedId: vi.fn(), getRootModules: vi.fn(), inlineScriptSession: { get: vi.fn(), set: vi.fn(), clear: vi.fn() }, @@ -853,19 +856,36 @@ describe('AIChatManager autonomy mode', () => { manager.isSessionChat = true manager.sessionId = 'htc1xouxd96dcyo6ruqo39' - manager.setFlowHelpers( - createFlowHelpers({ - hasPendingChanges: () => false, - acceptAllModuleActions: vi.fn(), - testFlow - }) - ) + manager.setFlowHelpers(createFlowHelpers({ testFlow })) manager.changeMode(AIMode.GLOBAL) - const jobId = await manager.helpers.testActiveFlow({ name: 'Ada' }) + const jobId = await manager.helpers.testActiveFlow('u/admin/live_flow', { name: 'Ada' }) expect(jobId).toBe('job-flow-preview') expect(testFlow).toHaveBeenCalledWith({ name: 'Ada' }) + // A session chat resolves an editor by its storage path, so it never names one. + expect(manager.flowAiChatHelpers).toBeUndefined() + }) + + // Session tabs keep every open flow editor mounted, so the last one to register is routinely + // a different flow than the one being tested. + it('tests the flow editor mounted on the storage path, not the last one registered', async () => { + const manager = new AIChatManager() + const testTarget = vi.fn(async () => 'job-target-flow') + const testLast = vi.fn(async () => 'job-last-flow') + + manager.setFlowHelpers( + createFlowHelpers({ testFlow: testTarget, storagePath: 'u/admin/live_flow' }) + ) + manager.setFlowHelpers( + createFlowHelpers({ testFlow: testLast, storagePath: 'u/admin/other_flow' }) + ) + + manager.changeMode(AIMode.GLOBAL) + const jobId = await manager.helpers.testActiveFlow('u/admin/live_flow', { name: 'Ada' }) + + expect(jobId).toBe('job-target-flow') + expect(testLast).not.toHaveBeenCalled() }) }) diff --git a/frontend/src/lib/components/copilot/chat/flow/FlowAIChat.svelte b/frontend/src/lib/components/copilot/chat/flow/FlowAIChat.svelte index 47a6b0b169..c6bed6988c 100644 --- a/frontend/src/lib/components/copilot/chat/flow/FlowAIChat.svelte +++ b/frontend/src/lib/components/copilot/chat/flow/FlowAIChat.svelte @@ -14,6 +14,9 @@ import type { ScriptLintResult } from '../shared' import { applyFlowJsonUpdate, updateRawScriptModuleContent } from './helperUtils' import { findModuleInFlow } from '$lib/components/flows/flowTree' + import { getEditorStoragePath } from '$lib/components/editorStoragePathContext' + + const editorStoragePath = getEditorStoragePath() let { flowModuleSchemaMap, @@ -162,6 +165,8 @@ selectionManager.selectId(id, { openPanel: true }) }, + getStoragePath: () => editorStoragePath?.(), + testFlow: async (args, conversationId) => { // Set preview args if provided if (args) { diff --git a/frontend/src/lib/components/copilot/chat/flow/core.ts b/frontend/src/lib/components/copilot/chat/flow/core.ts index 63c4ffe94f..51fea34d58 100644 --- a/frontend/src/lib/components/copilot/chat/flow/core.ts +++ b/frontend/src/lib/components/copilot/chat/flow/core.ts @@ -136,6 +136,10 @@ export interface FlowAIChatHelpers { /** Run a test of the current flow using the UI's preview mechanism */ testFlow: (args?: Record, conversationId?: string) => Promise + /** The path this editor's draft is stored under. Tells a caller which of several mounted + * editors is the one an active-editor context names. */ + getStoragePath: () => string | undefined + /** Get lint errors from a specific module (focuses it first, waits for Monaco to analyze) */ getLintErrors: (moduleId: string) => Promise } diff --git a/frontend/src/lib/components/copilot/chat/global/core.test.ts b/frontend/src/lib/components/copilot/chat/global/core.test.ts index d59b031947..1132c4c77f 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.test.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.test.ts @@ -4806,10 +4806,12 @@ describe('global AI tools', () => { }) }) - it('test_run_flow uses the live flow editor test hook when the active editor matches the path', async () => { + // The editor is driven by the key its draft is stored under, which reads and edits of the + // path resolve through too: a staged rename leaves that key where it was. + it('test_run_flow drives the live flow editor by its storage path', async () => { seedBackendDraft( 'flow', - '', + 'u/admin/live_flow_storage', { path: 'u/admin/live_flow', summary: 'Live flow', @@ -4825,7 +4827,7 @@ describe('global AI tools', () => { UserDraft.setLiveEditorDraft({ workspace: WORKSPACE, itemKind: 'flow', - storagePath: '', + storagePath: 'u/admin/live_flow_storage', effectivePath: 'u/admin/live_flow' }) const testActiveFlow = vi.fn(async () => 'job-live-flow') @@ -4842,7 +4844,7 @@ describe('global AI tools', () => { ) ) - expect(testActiveFlow).toHaveBeenCalledWith({ name: 'Ada' }) + expect(testActiveFlow).toHaveBeenCalledWith('u/admin/live_flow_storage', { name: 'Ada' }) expect(FlowService.getFlowByPath).not.toHaveBeenCalled() expect(JobService.runFlowPreview).not.toHaveBeenCalled() expect(result).toContain('Result (SUCCESS)') @@ -4851,7 +4853,7 @@ describe('global AI tools', () => { it('test_run_flow falls back to preview when the live flow editor test hook returns undefined', async () => { seedBackendDraft( 'flow', - '', + 'u/admin/live_flow_fallback', { path: 'u/admin/live_flow_fallback', summary: 'Live flow fallback', @@ -4867,7 +4869,7 @@ describe('global AI tools', () => { UserDraft.setLiveEditorDraft({ workspace: WORKSPACE, itemKind: 'flow', - storagePath: '', + storagePath: 'u/admin/live_flow_fallback', effectivePath: 'u/admin/live_flow_fallback' }) const testActiveFlow = vi.fn(async () => undefined) @@ -4884,7 +4886,7 @@ describe('global AI tools', () => { ) ) - expect(testActiveFlow).toHaveBeenCalledWith({ name: 'Ada' }) + expect(testActiveFlow).toHaveBeenCalledWith('u/admin/live_flow_fallback', { name: 'Ada' }) expect(FlowService.getFlowByPath).not.toHaveBeenCalled() expect(JobService.runFlowPreview).toHaveBeenCalledWith({ workspace: WORKSPACE, @@ -4896,6 +4898,52 @@ describe('global AI tools', () => { }) }) + // The flow may be open in a session tab that isn't the one on screen: driving its editor + // would paint the run into a tab the user is not looking at. + it('test_run_flow previews rather than driving an editor the user is not looking at', async () => { + seedBackendDraft( + 'flow', + 'u/admin/background_flow', + { + path: 'u/admin/background_flow', + summary: 'Background flow', + value: { modules: [{ id: 'background_step', value: { type: 'identity' } }] }, + schema: {}, + edited_by: '', + edited_at: '', + archived: false, + extra_perms: {} + }, + { workspace: WORKSPACE } + ) + UserDraft.setLiveEditorDraft({ + workspace: WORKSPACE, + itemKind: 'flow', + storagePath: 'u/admin/flow_on_screen', + effectivePath: 'u/admin/flow_on_screen' + }) + const testActiveFlow = vi.fn(async () => 'job-live-flow') + + await withCompletedTestJob(() => + callGlobalTool( + 'test_run_flow', + { path: 'u/admin/background_flow', args: { name: 'Ada' } }, + toolCallbacks, + { testActiveFlow } + ) + ) + + expect(testActiveFlow).not.toHaveBeenCalled() + expect(JobService.runFlowPreview).toHaveBeenCalledWith({ + workspace: WORKSPACE, + requestBody: { + path: 'u/admin/background_flow', + value: { modules: [{ id: 'background_step', value: { type: 'identity' } }] }, + args: { name: 'Ada' } + } + }) + }) + it('test_run_step previews rawscript steps from the draft flow', async () => { const content = 'export async function main(name: string) {\n\treturn name.toUpperCase()\n}' await callGlobalTool('write_flow', { diff --git a/frontend/src/lib/components/copilot/chat/global/core.ts b/frontend/src/lib/components/copilot/chat/global/core.ts index d22b69979f..868d0e1728 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.ts @@ -288,6 +288,8 @@ const ACTIVE_GLOBAL_EDITOR_DRAFTS: readonly { export type GlobalActiveEditorContext = { type: ActiveGlobalEditorType path: string + /** The key the draft is stored under, which `path` leaves behind on a rename. */ + storagePath: string isLiveDraft: true } @@ -4367,7 +4369,8 @@ type WriteDraftCtx = { export type SessionToolHelpers = { sessionId?: string } export type GlobalToolHelpers = SessionToolHelpers & { - testActiveFlow?: (args?: Record) => Promise + /** Runs the flow editor mounted on `storagePath`, if one is. */ + testActiveFlow?: (storagePath: string, args?: Record) => Promise attachedFiles?: AttachedFilesStore // Read/write the user-level Global instructions. `setUserInstructions` persists the // value and rebuilds the system message so the change applies on the next chat-loop @@ -4396,15 +4399,21 @@ function operatingWorkspaceFromHelpers(helpers: unknown): string | undefined { return (helpers as GlobalToolHelpers | undefined)?.operatingWorkspace } -function activeFlowTestFromCtx( +// Drive a live editor only for the flow on screen: several can be open at once (session tabs), +// and a run painted into a background tab is a side effect the user never sees. Undefined +// sends the caller to a preview run, which reports into the chat alone. The hook is bound to +// that editor's storage path — the same key reads and edits of `path` resolve through, so a +// staged rename cannot send the run to a different editor than the one being edited. +function liveFlowTestHookFromCtx( ctx: { workspace: string; helpers?: unknown }, path: string -): GlobalToolHelpers['testActiveFlow'] | undefined { +): ((args?: Record) => Promise) | undefined { const activeEditor = getActiveGlobalEditorContext(ctx.workspace) if (activeEditor?.type !== 'flow' || activeEditor.path !== path) { return undefined } - return (ctx.helpers as GlobalToolHelpers | undefined)?.testActiveFlow + const testActiveFlow = (ctx.helpers as GlobalToolHelpers | undefined)?.testActiveFlow + return testActiveFlow && ((args) => testActiveFlow(activeEditor.storagePath, args)) } export type OpenPreviewHandler = (req: { @@ -5775,7 +5784,7 @@ async function testRunFlowByPath( ): Promise { const { workspace, toolId, toolCallbacks } = ctx const testArgs = normalizeTestRunArgs(args.args) - const testActiveFlow = activeFlowTestFromCtx(ctx, args.path) + const testActiveFlow = liveFlowTestHookFromCtx(ctx, args.path) if (testActiveFlow) { return executeTestRun({ @@ -8171,9 +8180,10 @@ export function getActiveGlobalEditorContext( ): GlobalActiveEditorContext | undefined { for (const { itemKind, type } of ACTIVE_GLOBAL_EDITOR_DRAFTS) { const liveDraft = UserDraft.getLiveEditorDraft(itemKind, { workspace }) - const path = liveDraft?.effectivePath || liveDraft?.storagePath + if (!liveDraft) continue + const path = liveDraft.effectivePath || liveDraft.storagePath if (!path) continue - return { type, path, isLiveDraft: true } + return { type, path, storagePath: liveDraft.storagePath, isLiveDraft: true } } } diff --git a/frontend/src/lib/components/editorStoragePathContext.ts b/frontend/src/lib/components/editorStoragePathContext.ts new file mode 100644 index 0000000000..e3685995c4 --- /dev/null +++ b/frontend/src/lib/components/editorStoragePathContext.ts @@ -0,0 +1,19 @@ +import { getContext, setContext } from 'svelte' + +// The path an editor's draft is stored under, the same key the live-editor draft +// registers. Published by whichever ancestor owns it (the session tab, or the +// full-page editor) because the parts that must say which item an editor is open +// on sit below both, and the path they can see themselves is the renamed one. + +const KEY = 'EditorStoragePath' + +/** `undefined` where the editor has no stored draft to speak of (a drawer mount). */ +export type EditorStoragePath = () => string | undefined + +export function setEditorStoragePath(storagePath: EditorStoragePath): void { + setContext(KEY, storagePath) +} + +export function getEditorStoragePath(): EditorStoragePath | undefined { + return getContext(KEY) +} diff --git a/frontend/src/lib/components/sessions/SessionEditorTarget.svelte b/frontend/src/lib/components/sessions/SessionEditorTarget.svelte index 6e23d4e845..65e1a13abf 100644 --- a/frontend/src/lib/components/sessions/SessionEditorTarget.svelte +++ b/frontend/src/lib/components/sessions/SessionEditorTarget.svelte @@ -8,6 +8,7 @@ import { makeFlowCodec, makeScriptCodec, makeRawAppCodec } from './sessionDraftCodecs' import { itemDisplayName } from './previewRouter' import SessionItemNotFound from './SessionItemNotFound.svelte' + import { setEditorStoragePath } from '../editorStoragePathContext' let { runtime, @@ -51,6 +52,10 @@ // rely on its presence, not its identity. setContext('aiChatManager', runtime.manager) + // This tab's storage path, for the editor below: several tabs are mounted at + // once and only this one knows which item each is open on. + setEditorStoragePath(() => path) + // This tab's own editor cell (per (kind, path)); several tabs can be live at once. const cell = $derived( kind === 'flow' From 8bd144ccb154aa443d9f2832fda5497338734aaa Mon Sep 17 00:00:00 2001 From: Tristan TR <69242752+tristantr@users.noreply.github.com> Date: Thu, 10 Sep 2026 20:42:01 +0200 Subject: [PATCH 065/144] keep crawlers off the login page (#11057) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(frontend): keep crawlers off the login page Every page on the hub links to /user/login with itself in `rd`, so a crawler sees one login URL per hub page — 4,311 of them in Search Console, all rendering this same form and flagged as duplicates without a canonical. Nothing about a login page belongs in an index, on any instance. Mark the page noindex, as public_run already is, and ship a robots.txt that keeps crawlers out of /user/ and /api/. The frontend is embedded as static assets with an index.html fallback, which is why /robots.txt answered with the app shell until now; a real file in static/ is served as itself. Co-Authored-By: Claude Opus 5 * fix(frontend): let crawlers fetch the login page so the noindex is seen robots.txt disallowed /user/, which stopped a crawler fetching /user/login at all — and a page that is never fetched never shows its noindex. The two halves cancelled: the URLs would have moved from "duplicate" to "blocked" rather than out of the index. Drop the disallow, keeping /api/. And since the app is client-rendered, the meta tag only exists after a render pass; send X-Robots-Tag on /user/* from serve_path as well, which a crawler sees on the first fetch. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Opus 5 --- backend/windmill-api/src/static_assets.rs | 8 ++++++++ .../routes/(root)/(logged)/user/(user)/login/+page.svelte | 6 ++++++ frontend/static/robots.txt | 2 ++ 3 files changed, 16 insertions(+) create mode 100644 frontend/static/robots.txt diff --git a/backend/windmill-api/src/static_assets.rs b/backend/windmill-api/src/static_assets.rs index 06a79d022e..22645a2ac4 100644 --- a/backend/windmill-api/src/static_assets.rs +++ b/backend/windmill-api/src/static_assets.rs @@ -107,6 +107,14 @@ fn serve_path(path: &str, original_path: &str, query: Option<&str>) -> Response< .header("Cross-Origin-Resource-Policy", "cross-origin"); } + // Login and its siblings carry a different `rd` on every page that links + // to them, so a crawler meets thousands of URLs for one form. The app is + // client-rendered, so a meta tag only exists after a render pass; the + // header is seen on the first fetch. + if original_path.starts_with("/user/") { + res = res.header("X-Robots-Tag", "noindex, nofollow"); + } + // Add Content-Security-Policy header for static assets when policy is set if !CSP_POLICY.is_empty() { if let Ok(header_value) = HeaderValue::try_from(CSP_POLICY.as_str()) { diff --git a/frontend/src/routes/(root)/(logged)/user/(user)/login/+page.svelte b/frontend/src/routes/(root)/(logged)/user/(user)/login/+page.svelte index df85c1f1c1..e542201260 100644 --- a/frontend/src/routes/(root)/(logged)/user/(user)/login/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/user/(user)/login/+page.svelte @@ -141,6 +141,12 @@ } + + + + +
    diff --git a/frontend/static/robots.txt b/frontend/static/robots.txt new file mode 100644 index 0000000000..d34624918c --- /dev/null +++ b/frontend/static/robots.txt @@ -0,0 +1,2 @@ +User-agent: * +Disallow: /api/ From e62bfdcd8c6f1389601ccd2b5809c5eff0ff262e Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 10 Sep 2026 20:42:24 +0200 Subject: [PATCH 066/144] fix: give every table a primary key so the db can be logically replicated (#11036) * fix: give every table a primary key so the db can be logically replicated Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017WS9iNfYQiLJNuBnxysBzi * fix: tighten replicability guard and trim migration comments Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017WS9iNfYQiLJNuBnxysBzi * fix: split deployment_metadata into its own primary-key migration Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017WS9iNfYQiLJNuBnxysBzi * docs: correct the partial-index predicate note after the deployment_metadata split Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017WS9iNfYQiLJNuBnxysBzi --------- Co-authored-by: Claude Opus 5 (1M context) --- ...09052532_add_missing_primary_keys.down.sql | 9 ++++ ...0909052532_add_missing_primary_keys.up.sql | 31 +++++++++++++ ...d_deployment_metadata_primary_key.down.sql | 1 + ...add_deployment_metadata_primary_key.up.sql | 13 ++++++ ...909085549_add_metrics_primary_key.down.sql | 1 + ...60909085549_add_metrics_primary_key.up.sql | 12 +++++ backend/summarized_schema.txt | 14 +++--- .../tests/schema_replica_identity.rs | 46 +++++++++++++++++++ 8 files changed, 120 insertions(+), 7 deletions(-) create mode 100644 backend/migrations/20260909052532_add_missing_primary_keys.down.sql create mode 100644 backend/migrations/20260909052532_add_missing_primary_keys.up.sql create mode 100644 backend/migrations/20260909085548_add_deployment_metadata_primary_key.down.sql create mode 100644 backend/migrations/20260909085548_add_deployment_metadata_primary_key.up.sql create mode 100644 backend/migrations/20260909085549_add_metrics_primary_key.down.sql create mode 100644 backend/migrations/20260909085549_add_metrics_primary_key.up.sql create mode 100644 backend/windmill-common/tests/schema_replica_identity.rs diff --git a/backend/migrations/20260909052532_add_missing_primary_keys.down.sql b/backend/migrations/20260909052532_add_missing_primary_keys.down.sql new file mode 100644 index 0000000000..32ae30d2bd --- /dev/null +++ b/backend/migrations/20260909052532_add_missing_primary_keys.down.sql @@ -0,0 +1,9 @@ +-- Dropping the column drops the primary key and the identity sequence with it, and +-- only marks the column dropped in the catalog rather than rewriting the table, so +-- this takes the ACCESS EXCLUSIVE lock but not the time. + +ALTER TABLE workspace_runnable_dependencies DROP COLUMN IF EXISTS id; +ALTER TABLE dbt_node DROP COLUMN IF EXISTS id; +ALTER TABLE dbt_edge DROP COLUMN IF EXISTS id; +ALTER TABLE dbt_column_edge DROP COLUMN IF EXISTS id; +ALTER TABLE dbt_graph_snapshot DROP COLUMN IF EXISTS id; diff --git a/backend/migrations/20260909052532_add_missing_primary_keys.up.sql b/backend/migrations/20260909052532_add_missing_primary_keys.up.sql new file mode 100644 index 0000000000..399fe6cc5f --- /dev/null +++ b/backend/migrations/20260909052532_add_missing_primary_keys.up.sql @@ -0,0 +1,31 @@ +-- These five had neither a PRIMARY KEY nor an explicit REPLICA IDENTITY, which makes +-- PostgreSQL reject UPDATE and DELETE on them under logical replication. +-- `deployment_metadata` and `metrics` are the other two, one migration each after this. +-- +-- The surrogate cannot be swapped for a natural key: every unique index on all five is +-- PARTIAL, split on `script_hash IS NULL`, and a partial index cannot back a primary +-- key. The partial uniques stay; they are what the ON CONFLICT clauses infer. +-- +-- Each ALTER rewrites its table under ACCESS EXCLUSIVE and holds it unavailable for +-- the rewrite. These five share a transaction because each is bounded by what a +-- workspace holds rather than by how long it has run, so none can grow into the one +-- that locks the rest; a transaction holds all its locks until it commits. +-- +-- An instance that cannot afford that lock at startup can set REPLICA IDENTITY FULL +-- on these tables instead, which unblocks replication by itself, and run these +-- idempotent ALTERs in a maintenance window first. + +ALTER TABLE workspace_runnable_dependencies + ADD COLUMN IF NOT EXISTS id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY; + +ALTER TABLE dbt_node + ADD COLUMN IF NOT EXISTS id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY; + +ALTER TABLE dbt_edge + ADD COLUMN IF NOT EXISTS id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY; + +ALTER TABLE dbt_column_edge + ADD COLUMN IF NOT EXISTS id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY; + +ALTER TABLE dbt_graph_snapshot + ADD COLUMN IF NOT EXISTS id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY; diff --git a/backend/migrations/20260909085548_add_deployment_metadata_primary_key.down.sql b/backend/migrations/20260909085548_add_deployment_metadata_primary_key.down.sql new file mode 100644 index 0000000000..3ce4ae52a6 --- /dev/null +++ b/backend/migrations/20260909085548_add_deployment_metadata_primary_key.down.sql @@ -0,0 +1 @@ +ALTER TABLE deployment_metadata DROP COLUMN IF EXISTS id; diff --git a/backend/migrations/20260909085548_add_deployment_metadata_primary_key.up.sql b/backend/migrations/20260909085548_add_deployment_metadata_primary_key.up.sql new file mode 100644 index 0000000000..951253ce64 --- /dev/null +++ b/backend/migrations/20260909085548_add_deployment_metadata_primary_key.up.sql @@ -0,0 +1,13 @@ +-- The sixth of the seven; why any of them need a key is in +-- 20260909052532_add_missing_primary_keys. +-- +-- Kept out of that migration because it is the one table in the set with no retention +-- sweep -- rows accumulate per deployed script hash, flow version and app version for +-- the life of the instance -- so on an instance that upgrades after years of deploys +-- its ACCESS EXCLUSIVE rewrite is the one that could hold the others locked. +-- +-- No natural key: each row is a script, flow OR app deployment, and the three unique +-- indexes are partial on exactly that split, so none of them covers every row. + +ALTER TABLE deployment_metadata + ADD COLUMN IF NOT EXISTS id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY; diff --git a/backend/migrations/20260909085549_add_metrics_primary_key.down.sql b/backend/migrations/20260909085549_add_metrics_primary_key.down.sql new file mode 100644 index 0000000000..1ad428dc02 --- /dev/null +++ b/backend/migrations/20260909085549_add_metrics_primary_key.down.sql @@ -0,0 +1 @@ +ALTER TABLE metrics DROP COLUMN IF EXISTS row_id; diff --git a/backend/migrations/20260909085549_add_metrics_primary_key.up.sql b/backend/migrations/20260909085549_add_metrics_primary_key.up.sql new file mode 100644 index 0000000000..b863035990 --- /dev/null +++ b/backend/migrations/20260909085549_add_metrics_primary_key.up.sql @@ -0,0 +1,12 @@ +-- The last of the seven; why any of them need a key is in +-- 20260909052532_add_missing_primary_keys. +-- +-- Kept out of that migration because it is the largest (~400 MB / 750k rows on the +-- instance this was measured on, a steady state: `queue_%` rows, which are nearly all +-- of them, are swept at 14 days) and its ALTER rewrites it under ACCESS EXCLUSIVE. One +-- migration is one transaction, so alone it holds no lock on the others as it rewrites. +-- +-- The surrogate cannot be called `id` -- `metrics.id` holds the metric NAME. + +ALTER TABLE metrics + ADD COLUMN IF NOT EXISTS row_id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY; diff --git a/backend/summarized_schema.txt b/backend/summarized_schema.txt index a1e0b38518..d0f9d63647 100644 --- a/backend/summarized_schema.txt +++ b/backend/summarized_schema.txt @@ -70,15 +70,15 @@ ci_test_reference: workspace_id(char), test_script_path(char), test_script_hash( concurrency_settings: hash(bigint), concurrency_key(char), concurrent_limit(int), concurrency_time_window_s(int) config: name(char), config(jsonb) custom_concurrency_key_ended: key(char), ended_at(ts) -dbt_column_edge: workspace_id(char), script_path(char), script_hash(bigint), job_id(uuid), parent_unique_id(text), parent_column(text), child_unique_id(text), child_column(text), lineage_kind(text), ingested_at(ts) +dbt_column_edge: workspace_id(char), script_path(char), script_hash(bigint), job_id(uuid), parent_unique_id(text), parent_column(text), child_unique_id(text), child_column(text), lineage_kind(text), ingested_at(ts), id(bigint) FK: (workspace_id) -> workspace(id), (workspace_id, script_hash) -> script(workspace_id, hash) -dbt_edge: workspace_id(char), script_path(char), script_hash(bigint), job_id(uuid), parent_unique_id(text), child_unique_id(text), ingested_at(ts) +dbt_edge: workspace_id(char), script_path(char), script_hash(bigint), job_id(uuid), parent_unique_id(text), child_unique_id(text), ingested_at(ts), id(bigint) FK: (workspace_id) -> workspace(id), (workspace_id, script_hash) -> script(workspace_id, hash) -dbt_graph_snapshot: workspace_id(char), script_path(char), script_hash(bigint), job_id(uuid), digest(text), relation_root_at_last_ingest(text), ingested_at(ts), permissioned_as(char) +dbt_graph_snapshot: workspace_id(char), script_path(char), script_hash(bigint), job_id(uuid), digest(text), relation_root_at_last_ingest(text), ingested_at(ts), permissioned_as(char), id(bigint) FK: (workspace_id) -> workspace(id), (workspace_id, script_hash) -> script(workspace_id, hash) dbt_environment_state: workspace_id(char), script_path(char), environment(text), job_id(uuid), manifest(text), manifest_key(text), run_results(text), run_results_key(text), updated_at(ts) FK: (workspace_id) -> workspace(id) -dbt_node: workspace_id(char), script_path(char), script_hash(bigint), job_id(uuid), unique_id(text), resource_type(text), name(text), asset_path(text), materialized(text), materialize_strategy(text), unique_key(text), tags(text[]), description(text), test_kind(text), test_column(text), test_args(jsonb), severity(text), attached_node(text), columns(jsonb), column_schema(jsonb), freshness(jsonb), raw_code(text), original_file_path(text), ingested_at(ts) +dbt_node: workspace_id(char), script_path(char), script_hash(bigint), job_id(uuid), unique_id(text), resource_type(text), name(text), asset_path(text), materialized(text), materialize_strategy(text), unique_key(text), tags(text[]), description(text), test_kind(text), test_column(text), test_args(jsonb), severity(text), attached_node(text), columns(jsonb), column_schema(jsonb), freshness(jsonb), raw_code(text), original_file_path(text), ingested_at(ts), id(bigint) FK: (workspace_id) -> workspace(id), (workspace_id, script_hash) -> script(workspace_id, hash) dbt_run_progress: workspace_id(char), job_id(uuid), asset_kind(asset_kind), asset_path(char), status(materialization_status), row_count(bigint), error(text), updated_at(ts) FK: (workspace_id) -> workspace(id) @@ -88,7 +88,7 @@ debounce_key: key(char), job_id(uuid), previous_job_id(uuid), first_started_at(t debounce_stale_data: job_id(uuid), to_relock(text[]) debouncing_settings: hash(bigint), debounce_key(char), debounce_delay_s(int), max_total_debouncing_time(int), max_total_debounces_amount(int), debounce_args_to_accumulate(text[]) dependency_map: workspace_id(char), importer_path(char), importer_kind(importer_kind), imported_path(char), importer_node_id(char) -deployment_metadata: workspace_id(char), path(char), script_hash(bigint), app_version(bigint), callback_job_ids(uuid[]), deployment_msg(text), flow_version(bigint), job_id(uuid) +deployment_metadata: workspace_id(char), path(char), script_hash(bigint), app_version(bigint), callback_job_ids(uuid[]), deployment_msg(text), flow_version(bigint), job_id(uuid), id(bigint) FK: (workspace_id) -> workspace(id) draft: workspace_id(char), path(char), typ(draft_type), value(json), created_at(ts) FK: (workspace_id) -> workspace(id) @@ -147,7 +147,7 @@ mcp_oauth_refresh_token: id(bigint), refresh_token(char), access_token_hash(char mcp_oauth_server_client: client_id(char), client_name(char), redirect_uris(text[]), created_at(ts) mcp_oauth_server_code: code(char), client_id(char), user_email(char), workspace_id(char), scopes(text[]), redirect_uri(text), code_challenge(char), code_challenge_method(char), created_at(ts), expires_at(ts) FK: (client_id) -> mcp_oauth_server_client(client_id) -metrics: id(char), value(jsonb), created_at(ts) +metrics: id(char), value(jsonb), created_at(ts), row_id(bigint) mqtt_trigger: mqtt_resource_path(char), subscribe_topics(jsonb[]), client_version(mqtt_client_version), v5_config(jsonb), v3_config(jsonb), client_id(char), path(char), script_path(char), is_flow(bool), workspace_id(char), edited_by(char), email(char), edited_at(ts), extra_perms(jsonb), server_id(char), last_server_ping(ts), error(text), error_handler_path(char), error_handler_args(jsonb), retry(jsonb), mode(trigger_mode), labels(text[]) native_trigger: external_id(char), workspace_id(char), service_name(native_trigger_service), script_path(char), is_flow(bool), webhook_token_hash(char), service_config(jsonb), error(text), created_at(ts), updated_at(ts), enabled(bool) FK: (workspace_id) -> workspace(id) @@ -225,7 +225,7 @@ workspace_key: workspace_id(char), kind(workspace_key_kind), key(char) FK: (workspace_id) -> workspace(id) workspace_protection_rule: workspace_id(char), name(char), rules(int), bypass_groups(text[]), bypass_users(text[]), created_at(ts) FK: (workspace_id) -> workspace(id) -workspace_runnable_dependencies: flow_path(char), runnable_path(char), script_hash(bigint), runnable_is_flow(bool), workspace_id(char), app_path(char) +workspace_runnable_dependencies: flow_path(char), runnable_path(char), script_hash(bigint), runnable_is_flow(bool), workspace_id(char), app_path(char), id(bigint) FK: (app_path, workspace_id) -> app(path, workspace_id) | (flow_path, workspace_id) -> flow(path, workspace_id) workspace_settings: workspace_id(char), slack_team_id(char), slack_name(char), slack_command_script(char), slack_email(char), customer_id(char), plan(char), webhook(text), ai_config(jsonb), large_file_storage(jsonb), git_sync(jsonb), default_app(char), default_scripts(jsonb), deploy_ui(jsonb), mute_critical_alerts(bool), color(char), operator_settings(jsonb), teams_command_script(text), teams_team_id(text), teams_team_name(text), git_app_installations(jsonb), ducklake(jsonb), slack_oauth_client_id(char), slack_oauth_client_secret(char), datatable(jsonb), teams_team_guid(text), auto_invite(jsonb), error_handler(jsonb), success_handler(jsonb), public_app_execution_limit_per_minute(int), dbt_warehouses(jsonb), guest_access_enabled(bool), guest_jwt_public_key(text), guest_jwt_jwks_url(text) FK: (workspace_id) -> workspace(id) diff --git a/backend/windmill-common/tests/schema_replica_identity.rs b/backend/windmill-common/tests/schema_replica_identity.rs new file mode 100644 index 0000000000..fb14a5c78e --- /dev/null +++ b/backend/windmill-common/tests/schema_replica_identity.rs @@ -0,0 +1,46 @@ +//! Every table must be replicable. +//! +//! PostgreSQL refuses UPDATE and DELETE on a table that has neither a PRIMARY KEY +//! nor an explicit REPLICA IDENTITY once the database is published to a logical +//! replication slot. That is what a low-downtime major-version upgrade runs on +//! (RDS and Aurora Blue/Green, pglogical) and what every CDC pipeline reads, so a +//! single keyless table blocks the upgrade outright. This runs against a freshly +//! migrated database and fails on the migration that introduces one. + +use sqlx::{Pool, Postgres}; + +/// Partitioned parents are checked alongside ordinary tables: a parent without a +/// key hands the same defect to every partition created under it later. +#[sqlx::test(migrations = "../migrations")] +async fn every_table_is_replicable(db: Pool) -> anyhow::Result<()> { + let offenders: Vec = sqlx::query_scalar( + "SELECT n.nspname || '.' || c.relname + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE c.relkind IN ('r', 'p') + AND n.nspname NOT IN ('pg_catalog', 'information_schema') + AND NOT ( + -- FULL and USING INDEX replicate on their own. + c.relreplident IN ('f', 'i') + -- DEFAULT resolves to the primary key, so it needs one to exist. + -- NOTHING never replicates, primary key or not. + OR (c.relreplident = 'd' AND EXISTS ( + SELECT 1 FROM pg_index i WHERE i.indrelid = c.oid AND i.indisprimary + )) + ) + ORDER BY 1", + ) + .fetch_all(&db) + .await?; + + assert!( + offenders.is_empty(), + "logical replication will reject UPDATE and DELETE on these tables, because \ + they carry no replica identity it can use: {}. \ + Give each one a primary key -- a natural composite key where every column \ + is NOT NULL, otherwise a surrogate `BIGINT GENERATED ALWAYS AS IDENTITY`.", + offenders.join(", ") + ); + + Ok(()) +} From 63cb46d7bb9db1d996aa38e06bd3afebc60111bd Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 10 Sep 2026 20:50:09 +0200 Subject: [PATCH 067/144] feat: add a minimal skin for the approval page and slack/teams (#11061) * feat: add an approval skin to the approval page and slack/teams messages Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01KK2Ye4PizykReZVuMEm3m9 * chore: point ee-repo-ref at the teams approval skin commit Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01KK2Ye4PizykReZVuMEm3m9 * fix: resolve the approval skin from the step awaiting approval Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01KK2Ye4PizykReZVuMEm3m9 * fix: shorten the slack approval message to fit the button value limit Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01KK2Ye4PizykReZVuMEm3m9 * fix: rename skins to detailed/minimal and keep long slack messages Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01KK2Ye4PizykReZVuMEm3m9 * feat: title the minimal approval page from the step and flow summaries Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01KK2Ye4PizykReZVuMEm3m9 * feat: let wait_for_approval set the description approvers see Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01KK2Ye4PizykReZVuMEm3m9 * fix: keep a finished workflow's approval description, still gated Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01KK2Ye4PizykReZVuMEm3m9 * fix: keep a login-required approval locked after the run moves on Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01KK2Ye4PizykReZVuMEm3m9 * feat: hide the windmill version on the approval page Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01KK2Ye4PizykReZVuMEm3m9 * chore: update ee-repo-ref to e92abc9d1fba3ba898640df0cfeb0af8a50849b4 This commit updates the EE repository reference after PR #786 was merged in windmill-ee-private. Previous ee-repo-ref: bf1766ff49458f62d3f11746f1a06893ae3c2325 New ee-repo-ref: e92abc9d1fba3ba898640df0cfeb0af8a50849b4 Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: windmill-internal-app[bot] --- backend/ee-repo-ref.txt | 2 +- backend/windmill-api/openapi.yaml | 11 + backend/windmill-api/src/approvals.rs | 155 ++++++++---- backend/windmill-api/src/jobs.rs | 189 ++++++++++++-- backend/windmill-api/src/slack_approvals.rs | 230 ++++++++++++++---- backend/windmill-types/src/flows.rs | 31 +++ backend/windmill-worker/src/bun_executor.rs | 10 +- backend/windmill-worker/src/wac_executor.rs | 4 + cli/src/guidance/skills.gen.ts | 38 ++- docs/feature-telemetry.md | 2 +- .../src/lib/components/CenteredModal.svelte | 5 +- .../lib/components/InstanceSettings.svelte | 18 +- .../src/lib/components/LoginPageHeader.svelte | 13 +- .../approvals/MinimalApprovalSkin.svelte | 219 +++++++++++++++++ .../copilot/chat/flow/openFlow.json | 2 +- .../copilot/chat/flow/openFlowZod.gen.ts | 4 +- .../flows/content/FlowModuleSuspend.svelte | 39 +++ .../approve/[workspace]/[job]/+page.svelte | 33 ++- openflow.openapi.yaml | 8 + python-client/wmill/wmill/client.py | 19 +- system_prompts/auto-generated/flow.md | 2 +- system_prompts/auto-generated/prompts.ts | 26 +- system_prompts/auto-generated/script.md | 12 +- system_prompts/auto-generated/sdks/python.md | 6 +- .../auto-generated/sdks/typescript.md | 6 +- .../auto-generated/sdks/wac-python.md | 6 +- .../auto-generated/sdks/wac-typescript.md | 6 +- .../auto-generated/skills/write-flow/SKILL.md | 2 +- .../skills/write-script-bun/SKILL.md | 6 +- .../skills/write-script-bunnative/SKILL.md | 6 +- .../skills/write-script-deno/SKILL.md | 6 +- .../skills/write-script-python3/SKILL.md | 6 +- .../skills/write-workflow-as-code/SKILL.md | 12 +- typescript-client/client.ts | 10 + 34 files changed, 984 insertions(+), 160 deletions(-) create mode 100644 frontend/src/lib/components/approvals/MinimalApprovalSkin.svelte diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 975b266446..d426e06c2c 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -fe2418ff4e5630d6ad3fd85cd2c865bf51c87a2a +e92abc9d1fba3ba898640df0cfeb0af8a50849b4 diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 63adf55f51..09d0838def 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -16851,6 +16851,7 @@ paths: - can_approve - user_auth_required - approvers + - skin properties: flow_id: type: string @@ -16883,6 +16884,16 @@ paths: hide_cancel: type: boolean description: whether to hide the cancel button in the UI + skin: + type: string + enum: [detailed, minimal] + description: how the approval page presents the request + step_summary: + type: string + description: summary of the approval step, for the page title + flow_summary: + type: string + description: summary of the flow or workflow the approval belongs to approvers: type: array items: diff --git a/backend/windmill-api/src/approvals.rs b/backend/windmill-api/src/approvals.rs index e5b5f019eb..717be4ad0f 100644 --- a/backend/windmill-api/src/approvals.rs +++ b/backend/windmill-api/src/approvals.rs @@ -17,6 +17,7 @@ use std::str::FromStr; use uuid::Uuid; use windmill_common::cache; use windmill_common::error::Error; +use windmill_common::flows::{ApprovalSkin, Suspend}; use windmill_common::jobs::JobKind; use windmill_common::scripts::ScriptHash; @@ -94,6 +95,17 @@ pub struct ApprovalFormDetails { pub message_str: String, pub urls: ResumeUrls, pub schema: Option, + pub skin: ApprovalSkin, +} + +/// The suspended step an approval message is about, and the flow run it belongs to. +struct ApprovalStep { + created_by: String, + created_at: chrono::NaiveDateTime, + script_path: Option, + parent_job_id: Option, + args: Option>>, + suspend: Option, } #[allow(dead_code)] @@ -205,6 +217,90 @@ pub async fn get_approval_form_details( tracing::debug!("Job ID: {:?}", job_id); + let ApprovalStep { created_by, created_at, script_path, parent_job_id, args, suspend } = + fetch_approval_step(&db, w_id, job_id, flow_step_id).await?; + + let schema = suspend.as_ref().map(|suspend| ResumeFormRow { + resume_form: suspend.resume_form.clone(), + hide_cancel: suspend.hide_cancel, + }); + let skin = suspend.and_then(|s| s.skin).unwrap_or_default(); + + let bold_format = match format { + MessageFormat::Slack => "*{}*", + MessageFormat::Teams => "**{}**", + }; + + let message_str = match skin { + ApprovalSkin::Detailed => { + let args_str = args.map_or("None".to_string(), |a| { + serde_json::from_str::(a.get()) + .ok() + .and_then(|v| serde_json::to_string_pretty(&v).ok()) + .unwrap_or_else(|| a.get().to_string()) + }); + let parent_job_id_str = parent_job_id.map_or("None".to_string(), |id| id.to_string()); + let script_path_str = script_path.as_deref().unwrap_or("None"); + + let created_at_formatted = created_at.format("%Y-%m-%d %H:%M:%S").to_string(); + + let mut message_str = format!( + "A workflow has been suspended and is waiting for approval:\n\n\ + {}: {created_by}\n\n\ + {}: {created_at_formatted}\n\n\ + {}: {script_path_str}\n\n\ + {}:\n```\n{args_str}\n```\n\n\ + {}: {parent_job_id_str}\n\n", + bold_format.replace("{}", "Created by"), + bold_format.replace("{}", "Created at"), + bold_format.replace("{}", "Script path"), + bold_format.replace("{}", "Args"), + bold_format.replace("{}", "Flow ID") + ); + + // Append custom message if provided + if let Some(msg) = message { + message_str.push_str(msg); + } + message_str + } + ApprovalSkin::Minimal => format!( + "{}\n\n{}: {created_by}", + message.unwrap_or("Your approval is requested."), + bold_format.replace("{}", "Requested by"), + ), + }; + + tracing::debug!("Schema: {:#?}", schema); + + Ok(ApprovalFormDetails { message_str, urls, schema, skin }) +} + +/// The skin of the approval step `flow_step_id` of the flow running `job_id`. Falls back to +/// the detailed skin when the step cannot be resolved, so a message is still sent. +/// Reads through the unrestricted pool without an authorization check of its own: only the +/// skin, which is not sensitive, leaves this function. +pub(crate) async fn get_approval_step_skin( + db: &DB, + w_id: &str, + job_id: Uuid, + flow_step_id: &str, +) -> ApprovalSkin { + match fetch_approval_step(db, w_id, job_id, Some(flow_step_id)).await { + Ok(step) => step.suspend.and_then(|s| s.skin).unwrap_or_default(), + Err(e) => { + tracing::warn!("Could not resolve approval step {flow_step_id} of job {job_id}: {e}"); + ApprovalSkin::default() + } + } +} + +async fn fetch_approval_step( + db: &DB, + w_id: &str, + job_id: Uuid, + flow_step_id: Option<&str>, +) -> Result { // TODO: do we have a helper function for this? let (job_kind, script_hash, raw_flow, parent_job_id, created_at, created_by, script_path, args) = sqlx::query!( "WITH job_info AS ( @@ -240,17 +336,17 @@ pub async fn get_approval_form_details( job_id, &w_id ) - .fetch_optional(&db) + .fetch_optional(db) .await .map_err(|e| Error::BadRequest(e.to_string()))? .ok_or_else(|| Error::BadRequest("This workflow is no longer running and has either already timed out or been cancelled or completed.".to_string())) .map(|r| (r.job_kind, r.script_hash, r.raw_flow, r.parent_job, r.created_at, r.created_by, r.script_path, r.args))?; - let flow_data = match cache::job::fetch_flow(&db, &job_kind, script_hash).await { + let flow_data = match cache::job::fetch_flow(db, &job_kind, script_hash).await { Ok(data) => data, Err(_) => { if let Some(parent_job_id) = parent_job_id.as_ref() { - cache::job::fetch_preview_flow(&db, parent_job_id, raw_flow).await? + cache::job::fetch_preview_flow(db, parent_job_id, raw_flow).await? } else { return Err(Error::BadRequest( "This workflow is no longer running and has either already timed out or been cancelled or completed.".to_string(), @@ -265,49 +361,12 @@ pub async fn get_approval_form_details( tracing::debug!("Module: {:#?}", module); - let schema = module.and_then(|module| { - module.suspend.as_ref().map(|suspend| ResumeFormRow { - resume_form: suspend.resume_form.clone(), - hide_cancel: suspend.hide_cancel, - }) - }); - - let args_str = args.map_or("None".to_string(), |a| { - serde_json::from_str::(a.get()) - .ok() - .and_then(|v| serde_json::to_string_pretty(&v).ok()) - .unwrap_or_else(|| a.get().to_string()) - }); - let parent_job_id_str = parent_job_id.map_or("None".to_string(), |id| id.to_string()); - let script_path_str = script_path.as_deref().unwrap_or("None"); - - let created_at_formatted = created_at.format("%Y-%m-%d %H:%M:%S").to_string(); - - let bold_format = match format { - MessageFormat::Slack => "*{}*", - MessageFormat::Teams => "**{}**", - }; - - let mut message_str = format!( - "A workflow has been suspended and is waiting for approval:\n\n\ - {}: {created_by}\n\n\ - {}: {created_at_formatted}\n\n\ - {}: {script_path_str}\n\n\ - {}:\n```\n{args_str}\n```\n\n\ - {}: {parent_job_id_str}\n\n", - bold_format.replace("{}", "Created by"), - bold_format.replace("{}", "Created at"), - bold_format.replace("{}", "Script path"), - bold_format.replace("{}", "Args"), - bold_format.replace("{}", "Flow ID") - ); - - // Append custom message if provided - if let Some(msg) = message { - message_str.push_str(msg); - } - - tracing::debug!("Schema: {:#?}", schema); - - Ok(ApprovalFormDetails { message_str, urls, schema }) + Ok(ApprovalStep { + created_by, + created_at, + script_path, + parent_job_id, + args, + suspend: module.and_then(|m| m.suspend.clone()), + }) } diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 7c473486fd..84968ebbcc 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -107,7 +107,10 @@ use windmill_common::{ db::UserDB, error::{self, to_anyhow, Error}, flow_status::{Approval, ApprovalConditions, FlowStatus, FlowStatusModule}, - flows::{add_virtual_items_if_necessary, resolve_maybe_value, FlowValue}, + flows::{ + add_virtual_items_if_necessary, resolve_maybe_value, ApprovalSkin, FlowModule, FlowValue, + Suspend, + }, jobs::{script_path_to_payload, CompletedJob, JobKind, JobPayload, QueuedJob, RawCode}, oauth2::HmacSha256, query_builders, @@ -4858,6 +4861,11 @@ struct ApprovalInfo { user_auth_required: bool, #[serde(skip_serializing_if = "Option::is_none")] hide_cancel: Option, + skin: ApprovalSkin, + #[serde(skip_serializing_if = "Option::is_none")] + step_summary: Option, + #[serde(skip_serializing_if = "Option::is_none")] + flow_summary: Option, approvers: Vec, /// Share-read-link token for the flow, minted only for callers allowed to view this /// approval. Lets an authenticated workspace-member approver open the run details of @@ -4911,6 +4919,48 @@ fn can_approve_step( } } +/// The latest approval step the run has passed: a step before the current `step` that ran +/// rather than being skipped. Steps from `step` on don't count, because while an approval is +/// pending the step after it already holds the `WaitingForEvents` status. +fn last_reached_approval_step<'a>( + flow: &'a FlowValue, + status: &FlowStatus, +) -> Option<&'a FlowModule> { + flow.modules + .iter() + .zip(status.modules.iter()) + .take(usize::try_from(status.step).unwrap_or(0)) + .rev() + .filter(|(_, m)| matches!(m, FlowStatusModule::Success { skipped: false, .. })) + .map(|(module, _)| module) + .find(|module| module.suspend.is_some()) +} + +/// The approval conditions a step's own settings give, as the worker records them when the step +/// suspends. The worker drops them from the run once the step is approved, so a run that has +/// moved on is gated by these. Groups computed by an expression can't be re-evaluated outside +/// the run, so such a step falls back to any signed-in user. +fn approval_conditions_from_settings(suspend: &Suspend) -> Option { + let user_auth_required = suspend.user_auth_required.unwrap_or(false); + let self_approval_disabled = suspend.self_approval_disabled.unwrap_or(false); + if !user_auth_required && !self_approval_disabled { + return None; + } + let user_groups_required = match &suspend.user_groups_required { + Some(InputTransform::Static { value }) if user_auth_required => { + serde_json::from_str(value.get()).unwrap_or_default() + } + _ => vec![], + }; + Some(ApprovalConditions { user_auth_required, user_groups_required, self_approval_disabled }) +} + +/// How the approval step presents itself on the approval page. +struct ApprovalStepView { + skin: ApprovalSkin, + summary: Option, +} + async fn get_approval_info( OptAuthed(opt_authed): OptAuthed, Extension(db): Extension, @@ -4939,13 +4989,33 @@ async fn get_approval_info( script_path: Option, email: String, flow_status: Option, - workflow_as_code_status: Option, + // `v2_job_status` only holds a run that hasn't finished, so the fields below also read + // the completed run's status: a finished run's page keeps its skin and, for workflows as + // code, its description, still gated by the approval conditions the run had. + completed_flow_status: Option, + is_wac: bool, + wac_approval: Option, + approval_conditions: Option, + flow_summary: Option, } let row = sqlx::query_as::<_, ApprovalJobRow>( "SELECT j.id, j.runnable_path as script_path, j.permissioned_as_email as email, - s.flow_status, s.workflow_as_code_status + s.flow_status, + c.flow_status AS completed_flow_status, + COALESCE(s.workflow_as_code_status, c.workflow_as_code_status) IS NOT NULL + AS is_wac, + COALESCE(s.workflow_as_code_status, c.workflow_as_code_status)->'_approval' + AS wac_approval, + COALESCE(s.flow_status, c.flow_status)->'approval_conditions' + AS approval_conditions, + NULLIF(COALESCE(f.summary, sc.summary), '') AS flow_summary FROM v2_job j LEFT JOIN v2_job_status s ON s.id = j.id + LEFT JOIN v2_job_completed c ON c.id = j.id + LEFT JOIN flow f + ON j.kind = 'flow' AND f.workspace_id = j.workspace_id AND f.path = j.runnable_path + LEFT JOIN script sc + ON j.kind = 'script' AND sc.workspace_id = j.workspace_id AND sc.hash = j.runnable_id WHERE j.id = $1 AND j.workspace_id = $2", ) .bind(&job_id) @@ -4954,31 +5024,31 @@ async fn get_approval_info( .await? .ok_or_else(|| Error::NotFound(format!("Job {job_id} not found")))?; - let is_wac = row.workflow_as_code_status.is_some(); + let is_wac = row.is_wac; + let run_ac = row + .approval_conditions + .as_ref() + .and_then(|v| serde_json::from_value::(v.clone()).ok()); // Extract approval info based on WAC vs classic flow - let (form_schema, description, default_args, enums, approval_conditions, hide_cancel) = + let (form_schema, description, default_args, enums, approval_conditions, hide_cancel, step) = if is_wac { - let approval_meta = row - .workflow_as_code_status - .as_ref() - .and_then(|v| v.get("_approval")); + let approval_meta = row.wac_approval.as_ref(); let form = approval_meta.and_then(|m| m.get("form").cloned()); let default_args = approval_meta.and_then(|m| m.get("default_args").cloned()); let enums = approval_meta.and_then(|m| m.get("enums").cloned()); let description = approval_meta.and_then(|m| m.get("description").cloned()); - let ac = row - .flow_status - .as_ref() - .and_then(|v| v.get("approval_conditions")) - .and_then(|v| serde_json::from_value::(v.clone()).ok()); - (form, description, default_args, enums, ac, None) + let skin = approval_meta + .and_then(|m| m.get("skin")) + .and_then(|v| serde_json::from_value::(v.clone()).ok()) + .unwrap_or_default(); + let step = Some(ApprovalStepView { skin, summary: None }); + (form, description, default_args, enums, run_ac, None, step) } else { let fs = row .flow_status .as_ref() .and_then(|v| serde_json::from_value::(v.clone()).ok()); - let ac = fs.as_ref().and_then(|s| s.approval_conditions.clone()); // For classic flows, form/description come from the flow definition and step result let approval_step = fs.as_ref().map(|s| (s.step as usize).saturating_sub(1)); @@ -5038,6 +5108,28 @@ async fn get_approval_info( .and_then(|s| s.resume_form.as_ref()) .map(|rf| serde_json::json!(rf)); let hc = suspend_settings.map(|s| s.hide_cancel.unwrap_or(false)); + let completed_fs = row + .completed_flow_status + .as_ref() + .filter(|_| fs.is_none()) + .and_then(|v| serde_json::from_value::(v.clone()).ok()); + let approval_module = raw_flow + .as_ref() + .zip(fs.as_ref().or(completed_fs.as_ref())) + .and_then(|(flow, status)| last_reached_approval_step(flow, status)); + let ac = run_ac.or_else(|| { + approval_module + .and_then(|module| module.suspend.as_ref()) + .and_then(approval_conditions_from_settings) + }); + let step = approval_module.map(|module| ApprovalStepView { + skin: module + .suspend + .as_ref() + .and_then(|s| s.skin) + .unwrap_or_default(), + summary: module.summary.clone().filter(|s| !s.trim().is_empty()), + }); // Fetch description, default_args, and enums from the step's completed job result let step_job_id = fs @@ -5061,9 +5153,12 @@ async fn get_approval_info( (None, None, None) }; - (form, desc, default_args, enums, ac, hc) + (form, desc, default_args, enums, ac, hc, step) }; + let skin = step.as_ref().map(|s| s.skin).unwrap_or_default(); + let step_summary = step.and_then(|s| s.summary); + let user_auth_required = approval_conditions .as_ref() .map(|ac| ac.user_auth_required) @@ -5093,6 +5188,9 @@ async fn get_approval_info( can_approve: false, user_auth_required, hide_cancel: None, + skin, + step_summary: None, + flow_summary: None, approvers: vec![], view_token: None, })); @@ -5128,6 +5226,9 @@ async fn get_approval_info( can_approve, user_auth_required, hide_cancel, + skin, + step_summary, + flow_summary: row.flow_summary, approvers, view_token, })) @@ -11908,4 +12009,58 @@ mod approval_view_gate_tests { "trigger@example.com" )); } + + #[test] + fn approval_step_is_the_last_one_passed() { + let flow: FlowValue = serde_json::from_value(serde_json::json!({ "modules": [ + { "id": "a", "value": { "type": "identity" }, "suspend": {} }, + { "id": "b", "value": { "type": "identity" }, "suspend": {} }, + { "id": "c", "value": { "type": "identity" } } + ]})) + .unwrap(); + let step_at = |step: i32, types: [(&str, bool); 3]| { + let mut status = FlowStatus::new(&flow); + status.step = step; + status.modules = ["a", "b", "c"] + .into_iter() + .zip(types) + .map(|(id, (kind, skipped))| { + serde_json::from_value(serde_json::json!({ + "type": kind, "id": id, "job": Uuid::nil(), "count": 1, + "failed_retries": [], "skipped": skipped + })) + .unwrap() + }) + .collect(); + last_reached_approval_step(&flow, &status).map(|module| module.id.clone()) + }; + let waiting = ("WaitingForEvents", false); + let pending = ("WaitingForPriorSteps", false); + let ran = ("Success", false); + let skipped = ("Success", true); + // Awaiting a's approval: b, itself an approval step, already holds `WaitingForEvents`. + assert_eq!(step_at(1, [ran, waiting, pending]).as_deref(), Some("a")); + assert_eq!(step_at(2, [ran, ran, waiting]).as_deref(), Some("b")); + assert_eq!(step_at(3, [ran, skipped, ran]).as_deref(), Some("a")); + assert_eq!(step_at(0, [pending, pending, pending]), None); + } + + #[test] + fn approved_step_stays_gated_by_its_settings() { + let from_settings = |suspend: serde_json::Value| { + approval_conditions_from_settings(&serde_json::from_value(suspend).unwrap()) + }; + let login = from_settings(serde_json::json!({ + "user_auth_required": true, + "user_groups_required": { "type": "static", "value": ["approvers"] } + })); + assert!(!can_view( + &None, + &login, + Some("f/team/flow"), + "trigger@example.com" + )); + assert_eq!(login.unwrap().user_groups_required, ["approvers"]); + assert!(from_settings(serde_json::json!({})).is_none()); + } } diff --git a/backend/windmill-api/src/slack_approvals.rs b/backend/windmill-api/src/slack_approvals.rs index 593ea3b298..204b30010d 100644 --- a/backend/windmill-api/src/slack_approvals.rs +++ b/backend/windmill-api/src/slack_approvals.rs @@ -13,19 +13,25 @@ use sha2::Sha256; use sqlx::types::Uuid; use std::collections::HashMap; use windmill_common::error::{to_anyhow, Error}; +use windmill_common::flows::ApprovalSkin; +use windmill_common::utils::truncate_with_ellipsis; use windmill_common::variables::{get_secret_value_as_admin, get_workspace_key}; use crate::db::{ApiAuthed, DB}; use crate::jobs::{QueryApprover, ResumeUrls}; use crate::{ approvals::{ - extract_w_id_from_resume_url, handle_resume_action, ApprovalFormDetails, FieldType, - MessageFormat, QueryButtonText, QueryDefaultArgsJson, QueryDynamicEnumJson, - QueryFlowStepId, QueryMessage, ResumeFormField, ResumeSchema, + extract_w_id_from_resume_url, get_approval_step_skin, handle_resume_action, + ApprovalFormDetails, FieldType, MessageFormat, QueryButtonText, QueryDefaultArgsJson, + QueryDynamicEnumJson, QueryFlowStepId, QueryMessage, ResumeFormField, ResumeSchema, }, auth::OptTokened, }; +// Slack rejects a button value over 2000 characters, and with it the whole post. The button value +// carries the message on to the modal, so the message is shortened to fit. +const SLACK_BUTTON_VALUE_MAX_CHARS: usize = 2000; + #[derive(Deserialize, Debug)] pub struct SlackFormData { payload: String, @@ -127,6 +133,9 @@ struct PrivateMetadata { // HMAC over (w_id, resource_path) keyed on the workspace key; minted when the modal is // built, required by `handle_submission` before the resource_path is decrypted. signature: Option, + // Only selects the wording of the updated channel message, so it is left unsigned. + #[serde(default)] + skin: ApprovalSkin, } // Opportunistic transport-level check: when `SLACK_SIGNING_SECRET` is configured we verify @@ -432,6 +441,7 @@ async fn handle_submission( let container: Container = private_metadata.container; let hide_cancel = private_metadata.hide_cancel; let signature = private_metadata.signature; + let skin = private_metadata.skin; // If hide_cancel is true, we don't need to extract information from the private_metadata if hide_cancel.unwrap_or(false) && action == "cancel" { @@ -463,7 +473,7 @@ async fn handle_submission( tracing::warn!("Failed to resolve slack token for {w_id}/{resource_path}: {e:#}"); Error::BadRequest("Invalid Slack callback request".to_string()) })?; - update_original_slack_message(action, slack_token, container).await?; + update_original_slack_message(action, slack_token, container, skin).await?; Ok(()) } @@ -475,14 +485,19 @@ async fn transform_schemas( required: Option>, default_args_json: Option<&serde_json::Value>, dynamic_enums_json: Option<&serde_json::Value>, + skin: ApprovalSkin, ) -> Result { tracing::debug!("Resume urls: {:#?}", urls); + let link_label = match skin { + ApprovalSkin::Detailed => "Flow suspension details", + ApprovalSkin::Minimal => "View in Windmill", + }; let mut blocks = vec![serde_json::json!({ "type": "section", "text": { "type": "mrkdwn", - "text": format!("{}\n<{}|Flow suspension details>", text, urls.approvalPage), + "text": format!("{}\n<{}|{link_label}>", text, urls.approvalPage), } })]; @@ -918,10 +933,6 @@ async fn send_slack_message( value["approver"] = serde_json::json!(approver); } - if let Some(message) = message { - value["message"] = serde_json::json!(message); - } - if let Some(default_args_json) = default_args_json { value["default_args_json"] = default_args_json.clone(); } @@ -950,33 +961,8 @@ async fn send_slack_message( .map_err(|e| Box::new(e) as Box)?; value["signature"] = serde_json::json!(signature); - let payload = serde_json::json!({ - "channel": channel_id, - "text": "A flow has been suspended. Please approve or reject the flow.", - "blocks": [ - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "A flow has been suspended. Please approve or reject the flow." - } - }, - { - "type": "actions", - "elements": [ - { - "type": "button", - "text": { - "type": "plain_text", - "text": "View" - }, - "action_id": "open_modal", - "value": value.to_string() - } - ] - } - ] - }); + let skin = get_approval_step_skin(db, w_id, job_id, flow_step_id).await; + let payload = channel_message_payload(channel_id, skin, message, value); tracing::debug!("Payload: {:?}", payload); @@ -1000,6 +986,88 @@ async fn send_slack_message( Ok(StatusCode::OK) } +/// The channel post announcing the approval. Its button hands `button_value` to the modal, with +/// `message` added, shortened to what Slack's button value limit leaves room for. +fn channel_message_payload( + channel_id: &str, + skin: ApprovalSkin, + message: Option<&str>, + mut button_value: serde_json::Value, +) -> serde_json::Value { + let message = message.map(|m| message_fitting_button_value(&button_value, m)); + if let Some(message) = &message { + button_value["message"] = serde_json::json!(message); + } + let (text, section, button_label) = match skin { + ApprovalSkin::Detailed => { + let text = "A flow has been suspended. Please approve or reject the flow."; + (text, text.to_string(), "View") + } + ApprovalSkin::Minimal => { + let mut section = "*Approval requested*".to_string(); + if let Some(message) = &message { + section.push('\n'); + section.push_str(message); + } + ("Approval requested", section, "Review") + } + }; + + serde_json::json!({ + "channel": channel_id, + "text": text, + "blocks": [ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": section + } + }, + { + "type": "actions", + "elements": [ + { + "type": "button", + "text": { + "type": "plain_text", + "text": button_label + }, + "action_id": "open_modal", + "value": button_value.to_string() + } + ] + } + ] + }) +} + +/// The longest prefix of `message` that keeps `button_value` carrying it within Slack's limit. +fn message_fitting_button_value(button_value: &serde_json::Value, message: &str) -> String { + let mut with_message = button_value.clone(); + let mut fits = |max_chars: usize| { + let fitted = truncate_with_ellipsis(message, max_chars); + with_message["message"] = serde_json::json!(fitted); + (with_message.to_string().chars().count() <= SLACK_BUTTON_VALUE_MAX_CHARS).then_some(fitted) + }; + if let Some(whole) = fits(usize::MAX) { + return whole; + } + // Searched on the serialized length, which escaping makes longer than the raw prefix, and + // which grows with every character kept. + let (mut shortest, mut longest) = + (0, message.chars().count().min(SLACK_BUTTON_VALUE_MAX_CHARS)); + while shortest < longest { + let mid = (shortest + longest + 1) / 2; + if fits(mid).is_some() { + shortest = mid; + } else { + longest = mid - 1; + } + } + fits(shortest).unwrap_or_else(|| truncate_with_ellipsis(message, 0)) +} + async fn get_modal_blocks( db: DB, w_id: &str, @@ -1034,7 +1102,7 @@ async fn get_modal_blocks( ) .await?; - let ApprovalFormDetails { message_str, urls, schema } = approval_details; + let ApprovalFormDetails { message_str, urls, schema, skin } = approval_details; // Get the card content let card_content = transform_schemas( @@ -1063,6 +1131,7 @@ async fn get_modal_blocks( }), default_args_json, dynamic_enums_json, + skin, ) .await?; @@ -1077,6 +1146,7 @@ async fn get_modal_blocks( resume_button_text, cancel_button_text, &private_metadata_signature, + skin, ))) } @@ -1090,27 +1160,32 @@ fn construct_payload( resume_button_text: Option<&str>, cancel_button_text: Option<&str>, signature: &str, + skin: ApprovalSkin, ) -> serde_json::Value { + let (title, resume_label, cancel_label) = match skin { + ApprovalSkin::Detailed => ("Workflow Suspended", "Resume Workflow", "Cancel Workflow"), + ApprovalSkin::Minimal => ("Approval request", "Approve", "Reject"), + }; let mut view = serde_json::json!({ "type": "modal", "callback_id": "submit_form", "notify_on_close": true, "title": { "type": "plain_text", - "text": "Workflow Suspended" + "text": title }, "blocks": blocks, "submit": { "type": "plain_text", - "text": resume_button_text.unwrap_or("Resume Workflow") + "text": resume_button_text.unwrap_or(resume_label) }, - "private_metadata": serde_json::json!({ "resume_url": resume_url, "resource_path": resource_path, "container": container, "hide_cancel": hide_cancel, "signature": signature }).to_string(), + "private_metadata": serde_json::json!({ "resume_url": resume_url, "resource_path": resource_path, "container": container, "hide_cancel": hide_cancel, "signature": signature, "skin": skin }).to_string(), }); if !hide_cancel { view["close"] = serde_json::json!({ "type": "plain_text", - "text": cancel_button_text.unwrap_or("Cancel Workflow") + "text": cancel_button_text.unwrap_or(cancel_label) }); } @@ -1193,11 +1268,13 @@ async fn update_original_slack_message( action: &str, token: String, container: Container, + skin: ApprovalSkin, ) -> Result<(), Error> { - let message = if action == "resume" { - "\n\n*Workflow has been resumed!* :white_check_mark:" - } else { - "\n\n*Workflow has been canceled!* :x:" + let message = match (skin, action == "resume") { + (ApprovalSkin::Detailed, true) => "\n\n*Workflow has been resumed!* :white_check_mark:", + (ApprovalSkin::Detailed, false) => "\n\n*Workflow has been canceled!* :x:", + (ApprovalSkin::Minimal, true) => "*Approved* :white_check_mark:", + (ApprovalSkin::Minimal, false) => "*Rejected* :x:", }; let final_blocks = vec![serde_json::json!({ @@ -1242,3 +1319,66 @@ async fn update_original_slack_message( Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn long_message_keeps_the_channel_post_within_slack_limits() { + let button_value = serde_json::json!({ + "w_id": "demo", + "job_id": Uuid::nil(), + "path": "u/admin/slack", + "channel": "C0123456789", + "flow_step_id": "a", + "signature": "f".repeat(64), + }); + let carried = |skin, message: &str| { + let payload = channel_message_payload("C1", skin, Some(message), button_value.clone()); + let button = payload["blocks"][1]["elements"][0]["value"] + .as_str() + .unwrap() + .to_string(); + assert!(button.chars().count() <= SLACK_BUTTON_VALUE_MAX_CHARS); + let section = payload["blocks"][0]["text"]["text"].as_str().unwrap(); + assert!(section.chars().count() <= 3000); + serde_json::from_str::(&button) + .unwrap() + .message + .unwrap() + }; + // Quotes and newlines each cost two characters once escaped into the button value. + let message = "Expense \"offsite\" line\n".repeat(1000); + for skin in [ApprovalSkin::Detailed, ApprovalSkin::Minimal] { + let kept = carried(skin, &message); + let kept = kept.strip_suffix("...").unwrap(); + assert!(message.starts_with(kept)); + assert!(kept.chars().count() > 1_000); + assert_eq!(carried(skin, "Short message"), "Short message"); + } + } + + #[test] + fn minimal_skin_survives_the_modal_round_trip() { + let container = Container { message_ts: "1".to_string(), channel_id: "C1".to_string() }; + let payload = construct_payload( + serde_json::json!([]), + false, + "trigger", + "https://example.com/resume", + "u/admin/slack", + container, + None, + None, + "signature", + ApprovalSkin::Minimal, + ); + let view = &payload["view"]; + assert_eq!(view["submit"]["text"], "Approve"); + assert_eq!(view["close"]["text"], "Reject"); + let metadata: PrivateMetadata = + serde_json::from_str(view["private_metadata"].as_str().unwrap()).unwrap(); + assert_eq!(metadata.skin, ApprovalSkin::Minimal); + } +} diff --git a/backend/windmill-types/src/flows.rs b/backend/windmill-types/src/flows.rs index 02feeb421a..cb1c53a579 100644 --- a/backend/windmill-types/src/flows.rs +++ b/backend/windmill-types/src/flows.rs @@ -538,6 +538,20 @@ pub struct Suspend { pub hide_cancel: Option, #[serde(skip_serializing_if = "false_or_empty")] pub continue_on_disapprove_timeout: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub skin: Option, +} + +/// How an approval request is presented, on the approval page and in Slack/Teams messages. +#[derive(Deserialize, Serialize, Debug, Clone, Copy, PartialEq, Eq, Default)] +#[serde(rename_all = "lowercase")] +pub enum ApprovalSkin { + Minimal, + /// A skin this server does not know renders as the detailed one rather than failing to + /// deserialize the whole flow, so a flow authored against a newer version still runs. + #[default] + #[serde(other)] + Detailed, } fn false_or_empty(v: &Option) -> bool { @@ -1365,6 +1379,23 @@ mod tests { assert_eq!(val.modules.len(), 1); } + #[test] + fn suspend_skin_unknown_value_falls_back_to_detailed() { + let skin_of = |skin: &str| { + let val: FlowValue = serde_json::from_value(json!({ + "modules": [{ + "id": "a", + "value": {"type": "identity"}, + "suspend": {"required_events": 1, "skin": skin} + }] + })) + .unwrap(); + val.modules[0].suspend.as_ref().unwrap().skin + }; + assert_eq!(skin_of("minimal"), Some(ApprovalSkin::Minimal)); + assert_eq!(skin_of("not_a_skin_yet"), Some(ApprovalSkin::Detailed)); + } + #[test] fn agent_tool_keeps_description_through_locking() { // #10244: the dependency job rebuilds each tool from its locked FlowModule; the diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index a9844e5cfb..4b62dac902 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -1975,7 +1975,7 @@ async function run() {{ return {{ type: "inline_checkpoint", key: dispatch.key, result: dispatch.result ?? null, started_at: dispatch.started_at, duration_ms: dispatch.duration_ms }}; }} if (dispatch.mode === "approval") {{ - return {{ type: "approval", key: dispatch.key, timeout: dispatch.timeout, form: dispatch.form, self_approval_disabled: dispatch.self_approval_disabled }}; + return {{ type: "approval", key: dispatch.key, timeout: dispatch.timeout, form: dispatch.form, self_approval_disabled: dispatch.self_approval_disabled, skin: dispatch.skin, description: dispatch.description }}; }} if (dispatch.mode === "sleep") {{ return {{ type: "sleep", key: dispatch.key, seconds: dispatch.seconds }}; @@ -3206,7 +3206,7 @@ pub async fn handle_wac_v2_output( job.id, num_steps ))) } - WacOutput::Approval { key, timeout, form, self_approval_disabled } => { + WacOutput::Approval { key, timeout, form, self_approval_disabled, skin, description } => { let db = match conn { Connection::Sql(db) => db, _ => { @@ -3322,15 +3322,19 @@ pub async fn handle_wac_v2_output( }; // Store approval form metadata for the approval page endpoint - let approval_meta = serde_json::json!({ + let mut approval_meta = serde_json::json!({ "key": key, "form": form, "timeout": timeout_secs as u32, "self_approval_disabled": sad, + "skin": skin.unwrap_or_default(), "resume": resume_url, "cancel": cancel_url, "approvalPage": approval_page_url, }); + if let Some(description) = description.filter(|d| !d.is_null()) { + approval_meta["description"] = description; + } sqlx::query( "UPDATE v2_job_status SET workflow_as_code_status = jsonb_set( COALESCE(workflow_as_code_status, '{}'::jsonb), diff --git a/backend/windmill-worker/src/wac_executor.rs b/backend/windmill-worker/src/wac_executor.rs index cecf0e316c..798cd347b1 100644 --- a/backend/windmill-worker/src/wac_executor.rs +++ b/backend/windmill-worker/src/wac_executor.rs @@ -46,6 +46,10 @@ pub enum WacOutput { form: Option, #[serde(default)] self_approval_disabled: Option, + #[serde(default)] + skin: Option, + #[serde(default)] + description: Option, }, /// Server-side sleep — suspend the workflow for a duration without holding a worker. #[serde(rename = "sleep")] diff --git a/cli/src/guidance/skills.gen.ts b/cli/src/guidance/skills.gen.ts index 176d98eaa6..0c7b91887d 100644 --- a/cli/src/guidance/skills.gen.ts +++ b/cli/src/guidance/skills.gen.ts @@ -1037,12 +1037,16 @@ workflow(fn: (...args: any[]) => Promise): void * resume exactly this approval — route them through your own channel. Without a * key the steps are named \`approval\`, \`approval_2\`, ... * + * \`skin: "minimal"\` shows approvers only the request (form and approve/reject) + * instead of the detailed page with the workflow's details. \`description\` is + * shown above the form: a string, or a rich value such as \`{ markdown: "..." }\`. + * * @example * const urls = await step("urls", () => getApprovalUrls("manager")); * await step("notify", () => sendEmail(urls.resume, urls.cancel)); * const { value, approver } = await waitForApproval({ key: "manager", timeout: 3600 }); */ -waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; }): PromiseLike<{ value: any; approver: string; approved: boolean }> +waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; skin?: "detailed" | "minimal"; description?: string | object; }): PromiseLike<{ value: any; approver: string; approved: boolean }> /** * Resume/cancel/approval-page URLs bound to one \`waitForApproval\` step. @@ -1824,12 +1828,16 @@ workflow(fn: (...args: any[]) => Promise): void * resume exactly this approval — route them through your own channel. Without a * key the steps are named \`approval\`, \`approval_2\`, ... * + * \`skin: "minimal"\` shows approvers only the request (form and approve/reject) + * instead of the detailed page with the workflow's details. \`description\` is + * shown above the form: a string, or a rich value such as \`{ markdown: "..." }\`. + * * @example * const urls = await step("urls", () => getApprovalUrls("manager")); * await step("notify", () => sendEmail(urls.resume, urls.cancel)); * const { value, approver } = await waitForApproval({ key: "manager", timeout: 3600 }); */ -waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; }): PromiseLike<{ value: any; approver: string; approved: boolean }> +waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; skin?: "detailed" | "minimal"; description?: string | object; }): PromiseLike<{ value: any; approver: string; approved: boolean }> /** * Resume/cancel/approval-page URLs bound to one \`waitForApproval\` step. @@ -2705,12 +2713,16 @@ workflow(fn: (...args: any[]) => Promise): void * resume exactly this approval — route them through your own channel. Without a * key the steps are named \`approval\`, \`approval_2\`, ... * + * \`skin: "minimal"\` shows approvers only the request (form and approve/reject) + * instead of the detailed page with the workflow's details. \`description\` is + * shown above the form: a string, or a rich value such as \`{ markdown: "..." }\`. + * * @example * const urls = await step("urls", () => getApprovalUrls("manager")); * await step("notify", () => sendEmail(urls.resume, urls.cancel)); * const { value, approver } = await waitForApproval({ key: "manager", timeout: 3600 }); */ -waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; }): PromiseLike<{ value: any; approver: string; approved: boolean }> +waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; skin?: "detailed" | "minimal"; description?: string | object; }): PromiseLike<{ value: any; approver: string; approved: boolean }> /** * Resume/cancel/approval-page URLs bound to one \`waitForApproval\` step. @@ -4673,13 +4685,17 @@ async def sleep(seconds: int) # form: Optional form schema for the approval page. # self_approval: Whether the user who triggered the flow can approve it (default True). # key: Optional checkpoint key naming this approval step. +# skin: \`\`"minimal"\`\` shows approvers only the request (form and approve/reject) +# instead of the detailed page with the workflow's details. +# description: Shown to approvers above the form: a string, or a rich value such as +# \`\`{"markdown": "..."}\`\`. # # Example:: # # urls = await step("urls", lambda: get_approval_urls("manager")) # await step("notify", lambda: send_email(urls["resume"], urls["cancel"])) # result = await wait_for_approval(key="manager", timeout=3600) -async def wait_for_approval(timeout: int = 1800, form: dict | None = None, self_approval: bool = True, key: str | None = None) -> dict +async def wait_for_approval(timeout: int = 1800, form: dict | None = None, self_approval: bool = True, key: str | None = None, skin: Literal['detailed', 'minimal'] | None = None, description: str | dict | None = None) -> dict # Process items in parallel with optional concurrency control. # @@ -5558,7 +5574,7 @@ Reference a specific resource using \`$res:\` prefix: ## OpenFlow Schema -{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"Address of the account the flow runs on behalf of. Derived from on_behalf_of on read; accepted on write, where it is resolved to the account it names."},"on_behalf_of":{"type":"string","description":"The flow runs with the permissions of this identity: u/{username}, g/{group}, or a bare email when the username is itself email-shaped. The only stored half of the identity; on_behalf_of_email is derived from it. Omit it when writing and it is resolved from that address instead."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \\"yellow\\", \\"#ffff00\\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \\u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor"}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryAuto":{"type":"object","description":"Automatic context management","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","description":"Explicit message history","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Conversation memory configuration","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume","dbt"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'."},"description":{"type":"string","description":"Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script."},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\\nValid values: 'text' (default) - plain text response, 'image' - image generation\\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\\nFormat: Array<{ bucket: string, key: string }> - S3 object references\\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\\nRange: 0.0 to 2.0 (provider-dependent)\\n- 0.0 = deterministic, focused responses\\n- 0.7 = balanced (common default)\\n- 1.0+ = more creative/random\\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\\nRange: 1-1000.\\n"}},"required":["user_message"]},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default \`flow\`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"agent":{"type":"string","description":"Path of a reusable \`ai_agent\` resource (hybrid linking). When set, the agent brain\\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\\nthat resource; the module's input_transforms then only carry the flow-local inputs\\n(user_message/user_attachments).\\n"},"tool_inputs":{"type":"object","description":"Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\\nshared resource; overlaid onto the tools' input_transforms at runtime \\u2014 including when\\n\`agent\` is unset, since a step forked for editing keeps these overrides until it is saved\\nback or unlinked.\\n","additionalProperties":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}}},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}}`, +{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"Address of the account the flow runs on behalf of. Derived from on_behalf_of on read; accepted on write, where it is resolved to the account it names."},"on_behalf_of":{"type":"string","description":"The flow runs with the permissions of this identity: u/{username}, g/{group}, or a bare email when the username is itself email-shaped. The only stored half of the identity; on_behalf_of_email is derived from it. Omit it when writing and it is resolved from that address instead."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \\"yellow\\", \\"#ffff00\\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \\u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor"}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"},"skin":{"type":"string","enum":["detailed","minimal"],"description":"How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryAuto":{"type":"object","description":"Automatic context management","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","description":"Explicit message history","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Conversation memory configuration","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume","dbt"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'."},"description":{"type":"string","description":"Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script."},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\\nValid values: 'text' (default) - plain text response, 'image' - image generation\\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\\nFormat: Array<{ bucket: string, key: string }> - S3 object references\\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\\nRange: 0.0 to 2.0 (provider-dependent)\\n- 0.0 = deterministic, focused responses\\n- 0.7 = balanced (common default)\\n- 1.0+ = more creative/random\\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\\nRange: 1-1000.\\n"}},"required":["user_message"]},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default \`flow\`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"agent":{"type":"string","description":"Path of a reusable \`ai_agent\` resource (hybrid linking). When set, the agent brain\\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\\nthat resource; the module's input_transforms then only carry the flow-local inputs\\n(user_message/user_attachments).\\n"},"tool_inputs":{"type":"object","description":"Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\\nshared resource; overlaid onto the tools' input_transforms at runtime \\u2014 including when\\n\`agent\` is unset, since a step forked for editing keeps these overrides until it is saved\\nback or unlinked.\\n","additionalProperties":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}}},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}}`, "raw-app": `--- name: raw-app description: MUST use when creating raw apps. @@ -6794,12 +6810,16 @@ export async function sleep(seconds: number): Promise * resume exactly this approval — route them through your own channel. Without a * key the steps are named \`approval\`, \`approval_2\`, ... * + * \`skin: "minimal"\` shows approvers only the request (form and approve/reject) + * instead of the detailed page with the workflow's details. \`description\` is + * shown above the form: a string, or a rich value such as \`{ markdown: "..." }\`. + * * @example * const urls = await step("urls", () => getApprovalUrls("manager")); * await step("notify", () => sendEmail(urls.resume, urls.cancel)); * const { value, approver } = await waitForApproval({ key: "manager", timeout: 3600 }); */ -export function waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; }): PromiseLike<{ value: any; approver: string; approved: boolean }> +export function waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; skin?: "detailed" | "minimal"; description?: string | object; }): PromiseLike<{ value: any; approver: string; approved: boolean }> /** * Resume/cancel/approval-page URLs bound to one \`waitForApproval\` step. @@ -6975,13 +6995,17 @@ async def sleep(seconds: int) # form: Optional form schema for the approval page. # self_approval: Whether the user who triggered the flow can approve it (default True). # key: Optional checkpoint key naming this approval step. +# skin: \`\`"minimal"\`\` shows approvers only the request (form and approve/reject) +# instead of the detailed page with the workflow's details. +# description: Shown to approvers above the form: a string, or a rich value such as +# \`\`{"markdown": "..."}\`\`. # # Example:: # # urls = await step("urls", lambda: get_approval_urls("manager")) # await step("notify", lambda: send_email(urls["resume"], urls["cancel"])) # result = await wait_for_approval(key="manager", timeout=3600) -async def wait_for_approval(timeout: int = 1800, form: dict | None = None, self_approval: bool = True, key: str | None = None) -> dict +async def wait_for_approval(timeout: int = 1800, form: dict | None = None, self_approval: bool = True, key: str | None = None, skin: Literal['detailed', 'minimal'] | None = None, description: str | dict | None = None) -> dict # Get the resume/cancel/approval-page URLs bound to one \`\`wait_for_approval\`\` step. # diff --git a/docs/feature-telemetry.md b/docs/feature-telemetry.md index 9b4340e065..ac64800c36 100644 --- a/docs/feature-telemetry.md +++ b/docs/feature-telemetry.md @@ -4,7 +4,7 @@ anonymous usage-stats payload. It answers "does anyone use this, and which variant do they pick" without any identifying data leaving the instance. -It currently carries 48 registered actions across eighteen features (`ai_session`, `ai_chat`, +It currently carries 49 registered actions across eighteen features (`ai_session`, `ai_chat`, `ai_fix`, `ai_agent`, `ai_agent_eval`, `app_sandbox`, `datatable`, `flow_editor`, `flow_run`, `flow_step`, `home`, `run_form`, `debugger`, `trigger`, `command_script`, `hub_script`, `usage_meter`, `sso_groups_claim`). Nearly all of the diff --git a/frontend/src/lib/components/CenteredModal.svelte b/frontend/src/lib/components/CenteredModal.svelte index 3250b3a16a..08234c25b1 100644 --- a/frontend/src/lib/components/CenteredModal.svelte +++ b/frontend/src/lib/components/CenteredModal.svelte @@ -14,6 +14,8 @@ centerVertically?: boolean loading?: boolean containOverflow?: boolean + /** The Windmill version and update notice in the header. */ + showVersion?: boolean children?: import('svelte').Snippet } @@ -25,6 +27,7 @@ centerVertically = true, loading = false, containOverflow = false, + showVersion = true, children }: Props = $props() @@ -82,5 +85,5 @@ {/if}
    - + diff --git a/frontend/src/lib/components/InstanceSettings.svelte b/frontend/src/lib/components/InstanceSettings.svelte index 087e5e79eb..f965362659 100644 --- a/frontend/src/lib/components/InstanceSettings.svelte +++ b/frontend/src/lib/components/InstanceSettings.svelte @@ -1083,10 +1083,11 @@ loaded, whether SSO logins evaluate an IdP groups claim (SAML or OIDC) and change a membership, the plan tier and quota shown when the execution meter is opened, whether app sandbox isolation is turned on, whether a step's workspace script is edited from - the flow editor, how data tables and their migrations are set up and used, how often - an empty workspace home is seen, how often the home page’s create menu and hub-project - picker are opened and from which entry point, and the name of any public hub project - imported from the home page and how far that import got, last 30 days)
  • feature adoption (counts of which flow, script, trigger, worker and data table @@ -1146,10 +1147,11 @@ loaded, whether SSO logins evaluate an IdP groups claim (SAML or OIDC) and change a membership, the plan tier and quota shown when the execution meter is opened, whether app sandbox isolation is turned on, whether a step's workspace script is edited from - the flow editor, how data tables and their migrations are set up and used, how often - an empty workspace home is seen, how often the home page’s create menu and hub-project - picker are opened and from which entry point, and the name of any public hub project - imported from the home page and how far that import got, last 30 days)
  • feature adoption (counts of which flow, script, trigger, worker and data table diff --git a/frontend/src/lib/components/LoginPageHeader.svelte b/frontend/src/lib/components/LoginPageHeader.svelte index 7b49c97e34..ca5c7eafff 100644 --- a/frontend/src/lib/components/LoginPageHeader.svelte +++ b/frontend/src/lib/components/LoginPageHeader.svelte @@ -9,9 +9,10 @@ interface Props { /** Off for the login page, which puts the mark and the instance name in the middle. */ showBrand?: boolean + showVersion?: boolean } - let { showBrand = true }: Props = $props() + let { showBrand = true, showVersion = true }: Props = $props()
    @@ -31,9 +32,11 @@
    -
    - - -
    + {#if showVersion} +
    + + +
    + {/if}
    diff --git a/frontend/src/lib/components/approvals/MinimalApprovalSkin.svelte b/frontend/src/lib/components/approvals/MinimalApprovalSkin.svelte new file mode 100644 index 0000000000..a1a58a4ce5 --- /dev/null +++ b/frontend/src/lib/components/approvals/MinimalApprovalSkin.svelte @@ -0,0 +1,219 @@ + + +
    +
    +
    + {#if context} + + {context} + + {/if} + {#if job} +

    + Requested by {job.created_by} · +

    + {/if} +
    + {STATUS_BADGE[status].label} +
    + + {#if typeof approvalInfo.description === 'string'} +

    {approvalInfo.description}

    + {:else if approvalInfo.description != undefined} + + {/if} + + {#if status === 'pending'} + {#if hasForm} + {#if emptyString($enterpriseLicense)} + + {:else} + + {/if} + {/if} + + {#if approvalInfo.can_approve} +
    + {#if approvalInfo.hide_cancel !== true} + + {:else} +
    + {/if} + +
    + {#if isSelfApprovalBypass} + + As an administrator, by approving or rejecting this request, you bypass the self-approval + interdiction. + + {/if} + {:else if approvalInfo.user_auth_required && !$userStore} +

    Sign in to review this request.

    + + {:else} +
    +

    You are not authorized to approve this request.

    + {#if isSelfApprovalRefused} +

    Self-approval is disabled for this step.

    + {/if} + {#if groupsRequired.length > 0} +

    + Only members of the following groups can approve: + {groupsRequired.join(', ')} +

    + {/if} +
    + {/if} + {:else} +
    + {#if status === 'approved'} + + {:else if status === 'rejected'} + + {:else} + + {/if} +
    + + {status === 'closed' ? 'This request is closed' : STATUS_BADGE[status].label} + + + {#if status === 'approved'} + Your approval was recorded. You can close this page. + {:else if status === 'rejected'} + Your rejection was recorded. You can close this page. + {:else} + The flow is no longer waiting for approval. + {/if} + +
    +
    + {/if} + + {#if !isLocked && ((status === 'pending' && approvalInfo.approvers.length > 0) || isWorkspaceMember)} +
    + + {#if status === 'pending' && approvalInfo.approvers.length > 0} + Already approved by {approvalInfo.approvers.map((a) => a.approver).join(', ')} + {/if} + + {#if isWorkspaceMember} + + {/if} +
    + {/if} +
    diff --git a/frontend/src/lib/components/copilot/chat/flow/openFlow.json b/frontend/src/lib/components/copilot/chat/flow/openFlow.json index cab2839d48..7226329062 100644 --- a/frontend/src/lib/components/copilot/chat/flow/openFlow.json +++ b/frontend/src/lib/components/copilot/chat/flow/openFlow.json @@ -1 +1 @@ -{"openapi":"3.0.3","info":{"version":"1.791.0","title":"OpenFlow Spec","contact":{"name":"Ruben Fiszel","email":"ruben@windmill.dev","url":"https://windmill.dev"},"license":{"name":"Apache 2.0","url":"https://www.apache.org/licenses/LICENSE-2.0.html"}},"paths":{},"externalDocs":{"description":"documentation portal","url":"https://windmill.dev"},"components":{"schemas":{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"Address of the account the flow runs on behalf of. Derived from on_behalf_of on read; accepted on write, where it is resolved to the account it names."},"on_behalf_of":{"type":"string","description":"The flow runs with the permissions of this identity: u/{username}, g/{group}, or a bare email when the username is itself email-shaped. The only stored half of the identity; on_behalf_of_email is derived from it. Omit it when writing and it is resolved from that address instead."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \"yellow\", \"#ffff00\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution — modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor"}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type"}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type"}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryAuto":{"type":"object","description":"Automatic context management","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","description":"Explicit message history","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Conversation memory configuration","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind"}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type"}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type"}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume","dbt"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw",null]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw",null]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'."},"description":{"type":"string","description":"Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script."},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}]},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\nRange: 1-1000.\n"}},"required":["user_message"]},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default `flow`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"agent":{"type":"string","description":"Path of a reusable `ai_agent` resource (hybrid linking). When set, the agent brain\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\nthat resource; the module's input_transforms then only carry the flow-local inputs\n(user_message/user_attachments).\n"},"tool_inputs":{"type":"object","description":"Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\nshared resource; overlaid onto the tools' input_transforms at runtime — including when\n`agent` is unset, since a step forked for editing keeps these overrides until it is saved\nback or unlinked.\n","additionalProperties":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}}},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}}}} \ No newline at end of file +{"openapi":"3.0.3","info":{"version":"1.808.0","title":"OpenFlow Spec","contact":{"name":"Ruben Fiszel","email":"ruben@windmill.dev","url":"https://windmill.dev"},"license":{"name":"Apache 2.0","url":"https://www.apache.org/licenses/LICENSE-2.0.html"}},"paths":{},"externalDocs":{"description":"documentation portal","url":"https://windmill.dev"},"components":{"schemas":{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"Address of the account the flow runs on behalf of. Derived from on_behalf_of on read; accepted on write, where it is resolved to the account it names."},"on_behalf_of":{"type":"string","description":"The flow runs with the permissions of this identity: u/{username}, g/{group}, or a bare email when the username is itself email-shaped. The only stored half of the identity; on_behalf_of_email is derived from it. Omit it when writing and it is resolved from that address instead."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \"yellow\", \"#ffff00\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution — modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor"}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"},"skin":{"type":"string","enum":["detailed","minimal"],"description":"How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type"}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type"}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryAuto":{"type":"object","description":"Automatic context management","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","description":"Explicit message history","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Conversation memory configuration","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind"}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type"}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type"}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume","dbt"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw",null]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw",null]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'."},"description":{"type":"string","description":"Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script."},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}]},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\nRange: 1-1000.\n"}},"required":["user_message"]},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default `flow`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"agent":{"type":"string","description":"Path of a reusable `ai_agent` resource (hybrid linking). When set, the agent brain\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\nthat resource; the module's input_transforms then only carry the flow-local inputs\n(user_message/user_attachments).\n"},"tool_inputs":{"type":"object","description":"Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\nshared resource; overlaid onto the tools' input_transforms at runtime — including when\n`agent` is unset, since a step forked for editing keeps these overrides until it is saved\nback or unlinked.\n","additionalProperties":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}}},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}}}} \ No newline at end of file diff --git a/frontend/src/lib/components/copilot/chat/flow/openFlowZod.gen.ts b/frontend/src/lib/components/copilot/chat/flow/openFlowZod.gen.ts index 5bd36b85ce..f821413351 100644 --- a/frontend/src/lib/components/copilot/chat/flow/openFlowZod.gen.ts +++ b/frontend/src/lib/components/copilot/chat/flow/openFlowZod.gen.ts @@ -1,6 +1,6 @@ import { z } from "zod" -export const flowModuleValueSchema = z.discriminatedUnion("type", [z.object({ "input_transforms": z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs")).describe("Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments"), "content": z.string().describe("The script source code. Should export a 'main' function"), "language": z.enum(["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]).describe("Programming language for this script"), "path": z.string().describe("Optional path for saving this script").optional(), "lock": z.string().describe("Lock file content for dependencies").optional(), "type": z.literal("rawscript"), "tag": z.string().describe("Worker group tag for execution routing").optional(), "concurrent_limit": z.number().describe("Maximum concurrent executions of this script").optional(), "concurrency_time_window_s": z.number().describe("Time window for concurrent_limit").optional(), "custom_concurrency_key": z.string().describe("Custom key for grouping concurrent executions").optional(), "is_trigger": z.boolean().describe("If true, this script is a trigger that can start the flow").optional(), "assets": z.array(z.object({ "path": z.string().describe("Path to the asset"), "kind": z.enum(["s3object","resource","ducklake","datatable","volume","dbt"]).describe("Type of asset"), "access_type": z.union([z.literal("r"), z.literal("w"), z.literal("rw"), z.literal(null)]).nullable().describe("Access level for this asset").optional(), "alt_access_type": z.union([z.literal("r"), z.literal("w"), z.literal("rw"), z.literal(null)]).nullable().describe("Alternative access level").optional() })).describe("External resources this script accesses (S3 objects, resources, etc.)").optional() }).describe("Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms"), z.object({ "input_transforms": z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs")).describe("Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments"), "path": z.string().describe("Path to the script in the workspace (e.g., 'f/scripts/send_email')"), "hash": z.string().describe("Optional specific version hash of the script to use").optional(), "type": z.literal("script"), "tag_override": z.string().describe("Override the script's default worker group tag").optional(), "is_trigger": z.boolean().describe("If true, this script is a trigger that can start the flow").optional() }).describe("Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code"), z.object({ "input_transforms": z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs")).describe("Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments"), "path": z.string().describe("Path to the flow in the workspace (e.g., 'f/flows/process_user')"), "type": z.literal("flow") }).describe("Reference to an existing flow by path. Use this to call another flow as a subflow"), z.object({ "modules": z.array(z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.lazy(() => flowModuleValueSchema), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch")).describe("Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'"), "iterator": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs"), "skip_failures": z.boolean().describe("If true, iteration failures don't stop the loop. Failed iterations return null"), "type": z.literal("forloopflow"), "parallel": z.boolean().describe("If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency").optional(), "parallelism": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "squash": z.boolean().optional() }).describe("Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations"), z.object({ "modules": z.array(z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.lazy(() => flowModuleValueSchema), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch")).describe("Steps to execute in each iteration"), "skip_failures": z.boolean().describe("If true, iteration failures don't stop the loop. Failed iterations return null"), "type": z.literal("whileloopflow"), "parallel": z.boolean().describe("If true, iterations run concurrently (use with caution in while loops)").optional(), "parallelism": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "squash": z.boolean().optional() }).describe("Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result"), z.object({ "branches": z.array(z.object({ "summary": z.string().describe("Short description of this branch condition").optional(), "expr": z.string().describe("JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"), "modules": z.array(z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.lazy(() => flowModuleValueSchema), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch")).describe("Steps to execute if this branch's expr is true") })).describe("Array of branches to evaluate in order. The first branch with expr evaluating to true executes"), "default": z.array(z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.lazy(() => flowModuleValueSchema), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch")).describe("Steps to execute if no branch expressions match"), "type": z.literal("branchone") }).describe("Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes"), z.object({ "branches": z.array(z.object({ "summary": z.string().describe("Short description of this branch's purpose").optional(), "skip_failure": z.boolean().describe("If true, failure in this branch doesn't fail the entire flow").optional(), "modules": z.array(z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.lazy(() => flowModuleValueSchema), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch")).describe("Steps to execute in this branch") })).describe("Array of branches that all execute (either in parallel or sequentially)"), "type": z.literal("branchall"), "parallel": z.boolean().describe("If true, all branches execute concurrently. If false, they execute sequentially").optional() }).describe("Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently"), z.object({ "type": z.literal("identity"), "flow": z.boolean().describe("If true, marks this as a flow identity (special handling)").optional() }).describe("Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder"), z.object({ "input_transforms": z.object({ "provider": z.discriminatedUnion("type", [z.object({ "value": z.object({ "kind": z.enum(["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]).describe("Supported AI provider types"), "resource": z.string().describe("Resource reference in format '$res:{resource_path}' pointing to provider credentials"), "model": z.string().describe("Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"), "reasoning_effort": z.string().describe("Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default.").optional() }).describe("Complete AI provider configuration with resource reference and model selection"), "type": z.literal("static") }).describe("Static provider configuration passed directly to the AI agent"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined").optional(), "output_type": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n").optional(), "user_message": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."), "system_prompt": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("System instructions that guide the AI's behavior, persona, and response style. Optional.").optional(), "streaming": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n").optional(), "memory": z.discriminatedUnion("type", [z.object({ "value": z.discriminatedUnion("kind", [z.object({ "kind": z.literal("off") }).describe("No conversation memory/context"), z.object({ "kind": z.literal("auto"), "context_length": z.number().int().describe("Maximum number of messages to retain in context").optional(), "memory_id": z.string().describe("Identifier for persistent memory across agent invocations").optional() }).describe("Automatic context management"), z.object({ "kind": z.literal("manual"), "messages": z.array(z.object({ "role": z.enum(["user","assistant","system"]), "content": z.string() }).describe("A single message in conversation history")) }).describe("Explicit message history")]).describe("Conversation memory configuration"), "type": z.literal("static") }).describe("Static memory configuration passed directly to the AI agent"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined").optional(), "output_schema": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n").optional(), "user_attachments": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Array of file references (images or PDFs) for the AI agent.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\n").optional(), "max_completion_tokens": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n").optional(), "temperature": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n").optional(), "max_iterations": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Number. Limits how many times the agent can loop through reasoning and tool use.\nRange: 1-1000.\n").optional() }).describe("Input parameters for the AI agent mapped to their values"), "tools": z.array(z.object({ "id": z.string().describe("Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"), "summary": z.string().describe("The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'.").optional(), "description": z.string().describe("Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script.").optional(), "value": z.any().superRefine((x, ctx) => { +export const flowModuleValueSchema = z.discriminatedUnion("type", [z.object({ "input_transforms": z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs")).describe("Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments"), "content": z.string().describe("The script source code. Should export a 'main' function"), "language": z.enum(["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]).describe("Programming language for this script"), "path": z.string().describe("Optional path for saving this script").optional(), "lock": z.string().describe("Lock file content for dependencies").optional(), "type": z.literal("rawscript"), "tag": z.string().describe("Worker group tag for execution routing").optional(), "concurrent_limit": z.number().describe("Maximum concurrent executions of this script").optional(), "concurrency_time_window_s": z.number().describe("Time window for concurrent_limit").optional(), "custom_concurrency_key": z.string().describe("Custom key for grouping concurrent executions").optional(), "is_trigger": z.boolean().describe("If true, this script is a trigger that can start the flow").optional(), "assets": z.array(z.object({ "path": z.string().describe("Path to the asset"), "kind": z.enum(["s3object","resource","ducklake","datatable","volume","dbt"]).describe("Type of asset"), "access_type": z.union([z.literal("r"), z.literal("w"), z.literal("rw"), z.literal(null)]).nullable().describe("Access level for this asset").optional(), "alt_access_type": z.union([z.literal("r"), z.literal("w"), z.literal("rw"), z.literal(null)]).nullable().describe("Alternative access level").optional() })).describe("External resources this script accesses (S3 objects, resources, etc.)").optional() }).describe("Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms"), z.object({ "input_transforms": z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs")).describe("Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments"), "path": z.string().describe("Path to the script in the workspace (e.g., 'f/scripts/send_email')"), "hash": z.string().describe("Optional specific version hash of the script to use").optional(), "type": z.literal("script"), "tag_override": z.string().describe("Override the script's default worker group tag").optional(), "is_trigger": z.boolean().describe("If true, this script is a trigger that can start the flow").optional() }).describe("Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code"), z.object({ "input_transforms": z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs")).describe("Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments"), "path": z.string().describe("Path to the flow in the workspace (e.g., 'f/flows/process_user')"), "type": z.literal("flow") }).describe("Reference to an existing flow by path. Use this to call another flow as a subflow"), z.object({ "modules": z.array(z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.lazy(() => flowModuleValueSchema), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional(), "skin": z.enum(["detailed","minimal"]).describe("How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch")).describe("Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'"), "iterator": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs"), "skip_failures": z.boolean().describe("If true, iteration failures don't stop the loop. Failed iterations return null"), "type": z.literal("forloopflow"), "parallel": z.boolean().describe("If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency").optional(), "parallelism": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "squash": z.boolean().optional() }).describe("Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations"), z.object({ "modules": z.array(z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.lazy(() => flowModuleValueSchema), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional(), "skin": z.enum(["detailed","minimal"]).describe("How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch")).describe("Steps to execute in each iteration"), "skip_failures": z.boolean().describe("If true, iteration failures don't stop the loop. Failed iterations return null"), "type": z.literal("whileloopflow"), "parallel": z.boolean().describe("If true, iterations run concurrently (use with caution in while loops)").optional(), "parallelism": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "squash": z.boolean().optional() }).describe("Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result"), z.object({ "branches": z.array(z.object({ "summary": z.string().describe("Short description of this branch condition").optional(), "expr": z.string().describe("JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"), "modules": z.array(z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.lazy(() => flowModuleValueSchema), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional(), "skin": z.enum(["detailed","minimal"]).describe("How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch")).describe("Steps to execute if this branch's expr is true") })).describe("Array of branches to evaluate in order. The first branch with expr evaluating to true executes"), "default": z.array(z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.lazy(() => flowModuleValueSchema), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional(), "skin": z.enum(["detailed","minimal"]).describe("How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch")).describe("Steps to execute if no branch expressions match"), "type": z.literal("branchone") }).describe("Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes"), z.object({ "branches": z.array(z.object({ "summary": z.string().describe("Short description of this branch's purpose").optional(), "skip_failure": z.boolean().describe("If true, failure in this branch doesn't fail the entire flow").optional(), "modules": z.array(z.object({ "id": z.string().describe("Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"), "value": z.lazy(() => flowModuleValueSchema), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional(), "skin": z.enum(["detailed","minimal"]).describe("How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch")).describe("Steps to execute in this branch") })).describe("Array of branches that all execute (either in parallel or sequentially)"), "type": z.literal("branchall"), "parallel": z.boolean().describe("If true, all branches execute concurrently. If false, they execute sequentially").optional() }).describe("Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently"), z.object({ "type": z.literal("identity"), "flow": z.boolean().describe("If true, marks this as a flow identity (special handling)").optional() }).describe("Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder"), z.object({ "input_transforms": z.object({ "provider": z.discriminatedUnion("type", [z.object({ "value": z.object({ "kind": z.enum(["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]).describe("Supported AI provider types"), "resource": z.string().describe("Resource reference in format '$res:{resource_path}' pointing to provider credentials"), "model": z.string().describe("Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"), "reasoning_effort": z.string().describe("Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default.").optional() }).describe("Complete AI provider configuration with resource reference and model selection"), "type": z.literal("static") }).describe("Static provider configuration passed directly to the AI agent"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined").optional(), "output_type": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n").optional(), "user_message": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."), "system_prompt": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("System instructions that guide the AI's behavior, persona, and response style. Optional.").optional(), "streaming": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n").optional(), "memory": z.discriminatedUnion("type", [z.object({ "value": z.discriminatedUnion("kind", [z.object({ "kind": z.literal("off") }).describe("No conversation memory/context"), z.object({ "kind": z.literal("auto"), "context_length": z.number().int().describe("Maximum number of messages to retain in context").optional(), "memory_id": z.string().describe("Identifier for persistent memory across agent invocations").optional() }).describe("Automatic context management"), z.object({ "kind": z.literal("manual"), "messages": z.array(z.object({ "role": z.enum(["user","assistant","system"]), "content": z.string() }).describe("A single message in conversation history")) }).describe("Explicit message history")]).describe("Conversation memory configuration"), "type": z.literal("static") }).describe("Static memory configuration passed directly to the AI agent"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined").optional(), "output_schema": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n").optional(), "user_attachments": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Array of file references (images or PDFs) for the AI agent.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\n").optional(), "max_completion_tokens": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n").optional(), "temperature": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n").optional(), "max_iterations": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").describe("Number. Limits how many times the agent can loop through reasoning and tool use.\nRange: 1-1000.\n").optional() }).describe("Input parameters for the AI agent mapped to their values"), "tools": z.array(z.object({ "id": z.string().describe("Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"), "summary": z.string().describe("The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'.").optional(), "description": z.string().describe("Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script.").optional(), "value": z.any().superRefine((x, ctx) => { const schemas = [z.intersection(z.object({ "tool_type": z.literal("flowmodule") }), z.lazy(() => flowModuleValueSchema)).describe("A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module"), z.object({ "tool_type": z.literal("mcp"), "resource_path": z.string().describe("Path to the MCP resource/server configuration"), "include_tools": z.array(z.string()).describe("Whitelist of specific tools to include from this MCP server").optional(), "exclude_tools": z.array(z.string()).describe("Blacklist of tools to exclude from this MCP server").optional() }).describe("Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers"), z.object({ "tool_type": z.literal("websearch") }).describe("A tool implemented as a websearch tool. The AI can call this like any other websearch tool")]; const errors = schemas.reduce( (errors, schema) => @@ -37,7 +37,7 @@ export const flowModuleSchema = z.object({ "id": z.string().describe("Unique ide message: "Invalid input: Should pass single schema", }); } - }).describe("The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference") }).describe("A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool")).describe("Array of tools the agent can use. The agent decides which tools to call based on the task").optional(), "type": z.literal("aiagent"), "tag": z.string().describe("Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default `flow`)").optional(), "omit_output_from_conversation": z.boolean().describe("If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled.").default(false), "agent": z.string().describe("Path of a reusable `ai_agent` resource (hybrid linking). When set, the agent brain\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\nthat resource; the module's input_transforms then only carry the flow-local inputs\n(user_message/user_attachments).\n").optional(), "tool_inputs": z.record(z.string(), z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs"))).describe("Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\nshared resource; overlaid onto the tools' input_transforms at runtime — including when\n`agent` is unset, since a step forked for editing keeps these overrides until it is saved\nback or unlinked.\n").optional(), "parallel": z.boolean().describe("If true, the agent can execute multiple tool calls in parallel").optional() }).describe("AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task")]).describe("The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type"), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch") + }).describe("The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference") }).describe("A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool")).describe("Array of tools the agent can use. The agent decides which tools to call based on the task").optional(), "type": z.literal("aiagent"), "tag": z.string().describe("Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default `flow`)").optional(), "omit_output_from_conversation": z.boolean().describe("If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled.").default(false), "agent": z.string().describe("Path of a reusable `ai_agent` resource (hybrid linking). When set, the agent brain\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\nthat resource; the module's input_transforms then only carry the flow-local inputs\n(user_message/user_attachments).\n").optional(), "tool_inputs": z.record(z.string(), z.record(z.string(), z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs"))).describe("Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\nshared resource; overlaid onto the tools' input_transforms at runtime — including when\n`agent` is unset, since a step forked for editing keeps these overrides until it is saved\nback or unlinked.\n").optional(), "parallel": z.boolean().describe("If true, the agent can execute multiple tool calls in parallel").optional() }).describe("AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task")]).describe("The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type"), "stop_after_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "stop_after_all_iters_if": z.object({ "skip_if_stopped": z.boolean().describe("If true, following steps are skipped when this condition triggers").optional(), "expr": z.string().describe("JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"), "error_message": z.string().nullable().describe("Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised.").optional(), "error_include_result": z.boolean().describe("When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false.").optional() }).describe("Early termination condition for a module").optional(), "skip_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'") }).describe("Conditionally skip this step based on previous results or flow inputs").optional(), "sleep": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "cache_ttl": z.number().describe("Cache duration in seconds for this step's results").optional(), "cache_ignore_s3_path": z.boolean().optional(), "timeout": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "delete_after_secs": z.number().int().describe("If set, delete the step's args, result and logs after this many seconds following job completion").optional(), "summary": z.string().describe("Short description of what this step does").optional(), "mock": z.object({ "enabled": z.boolean().describe("If true, return mock value instead of executing").optional(), "return_value": z.any().describe("Value to return when mocked").optional() }).describe("Mock configuration for testing without executing the actual step").optional(), "suspend": z.object({ "required_events": z.number().int().describe("Number of approvals required before continuing").optional(), "timeout": z.number().int().describe("Timeout in seconds before auto-continuing or canceling").optional(), "resume_form": z.object({ "schema": z.record(z.string(), z.any()).describe("JSON Schema for the resume form").optional() }).describe("Form schema for collecting input when resuming").optional(), "user_auth_required": z.boolean().describe("If true, only authenticated users can approve").optional(), "user_groups_required": z.discriminatedUnion("type", [z.object({ "value": z.any().describe("The static value. For resources, use format '$res:path/to/resource'").optional(), "type": z.literal("static") }).describe("Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'"), z.object({ "expr": z.string().describe("JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"), "type": z.literal("javascript") }).describe("JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')"), z.object({ "type": z.literal("ai") }).describe("Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.")]).describe("Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs").optional(), "self_approval_disabled": z.boolean().describe("If true, the user who started the flow cannot approve").optional(), "hide_cancel": z.boolean().describe("If true, hide the cancel button on the approval form").optional(), "continue_on_disapprove_timeout": z.boolean().describe("If true, continue flow on timeout instead of canceling").optional(), "skin": z.enum(["detailed","minimal"]).describe("How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions").optional() }).describe("Configuration for approval/resume steps that wait for user input").optional(), "priority": z.number().describe("Execution priority for this step (higher numbers run first)").optional(), "continue_on_error": z.boolean().describe("If true, flow continues even if this step fails").optional(), "retry": z.object({ "constant": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "seconds": z.number().int().describe("Seconds to wait between retries").optional() }).describe("Retry with constant delay between attempts").optional(), "exponential": z.object({ "attempts": z.number().int().describe("Number of retry attempts").optional(), "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() }).describe("Retry with exponential backoff (delay doubles each time)").optional(), "retry_if": z.object({ "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") }).describe("Conditional retry based on error or result").optional() }).describe("Retry configuration for failed module executions").optional(), "debouncing": z.object({ "debounce_delay_s": z.number().int().describe("Delay in seconds to debounce this step's executions across flow runs").optional(), "debounce_key": z.string().describe("Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-").optional(), "debounce_args_to_accumulate": z.array(z.string()).describe("Array-type arguments to accumulate across debounced executions").optional(), "max_total_debouncing_time": z.number().int().describe("Maximum total time in seconds before forced execution").optional(), "max_total_debounces_amount": z.number().int().describe("Maximum number of debounces before forced execution").optional() }).describe("Debounce configuration for this step (EE only)").optional() }).describe("A single step in a flow. Can be a script, subflow, loop, or branch") export const flowModulesSchema = z.array(flowModuleSchema) diff --git a/frontend/src/lib/components/flows/content/FlowModuleSuspend.svelte b/frontend/src/lib/components/flows/content/FlowModuleSuspend.svelte index 6081e30846..1230de1b05 100644 --- a/frontend/src/lib/components/flows/content/FlowModuleSuspend.svelte +++ b/frontend/src/lib/components/flows/content/FlowModuleSuspend.svelte @@ -16,8 +16,13 @@ import SuspendDrawer from './SuspendDrawer.svelte' import EditableSchemaDrawer from '$lib/components/schema/EditableSchemaDrawer.svelte' import SchemaForm from '$lib/components/SchemaForm.svelte' + import ToggleButtonGroup from '$lib/components/common/toggleButton-v2/ToggleButtonGroup.svelte' + import ToggleButton from '$lib/components/common/toggleButton-v2/ToggleButton.svelte' import { Pen, Plus } from 'lucide-svelte' import { slideDynamic } from '$lib/transitions' + import { logFeatureUsage } from '$lib/utils/featureUsage' + + type ApprovalSkin = NonNullable['skin']> const { selectionManager, flowStateStore, opWorkspace } = getContext('FlowEditorContext') @@ -81,6 +86,12 @@ } formEditor?.openDrawer() } + + function setSkin(skin: ApprovalSkin) { + if (!flowModule.suspend) return + flowModule.suspend.skin = skin === 'detailed' ? undefined : skin + logFeatureUsage('flow_step', 'approval_skin', { key: skin }) + }
    @@ -134,6 +145,34 @@ {/if} + { if (!job?.args) return job?.args const args = { ...(job.args as any) } @@ -211,8 +221,10 @@ {#if error}
    @@ -237,6 +249,25 @@

    {error}

    {/if}
    + {:else if approvalInfo && skin === 'minimal'} + {:else if approvalInfo} {#if !isLocked}
    diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index 32da611f6b..1d7af0ebb0 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -362,6 +362,14 @@ components: continue_on_disapprove_timeout: type: boolean description: If true, continue flow on timeout instead of canceling + skin: + type: string + enum: [detailed, minimal] + description: >- + How the approval request is presented, on the approval page and in Slack/Teams + approval messages. 'detailed' (used when unset) shows the flow details + (arguments, graph, approvers); 'minimal' shows only the request: the step + description, form and approve/reject actions priority: type: number description: Execution priority for this step (higher numbers run first) diff --git a/python-client/wmill/wmill/client.py b/python-client/wmill/wmill/client.py index 98bbd7c9f3..918a16f2b0 100644 --- a/python-client/wmill/wmill/client.py +++ b/python-client/wmill/wmill/client.py @@ -3085,6 +3085,8 @@ class WorkflowCtx: form: dict | None = None, self_approval: bool = True, key: str | None = None, + skin: str | None = None, + description: str | dict | None = None, ): if key is not None: _assert_usable_step_key(key, "wait_for_approval key") @@ -3113,6 +3115,8 @@ class WorkflowCtx: "timeout": timeout, "form": form, "self_approval_disabled": not self_approval, + "skin": skin, + "description": description, "steps": [], }) @@ -3559,6 +3563,8 @@ async def wait_for_approval( form: dict | None = None, self_approval: bool = True, key: str | None = None, + skin: Literal["detailed", "minimal"] | None = None, + description: str | dict | None = None, ) -> dict: """Suspend the workflow and wait for an external approval. @@ -3573,6 +3579,10 @@ async def wait_for_approval( form: Optional form schema for the approval page. self_approval: Whether the user who triggered the flow can approve it (default True). key: Optional checkpoint key naming this approval step. + skin: ``"minimal"`` shows approvers only the request (form and approve/reject) + instead of the detailed page with the workflow's details. + description: Shown to approvers above the form: a string, or a rich value such as + ``{"markdown": "..."}``. Example:: @@ -3583,7 +3593,12 @@ async def wait_for_approval( ctx: WorkflowCtx | None = _workflow_ctx.get(None) if ctx is not None: return await ctx._wait_for_approval( - timeout=timeout, form=form, self_approval=self_approval, key=key + timeout=timeout, + form=form, + self_approval=self_approval, + key=key, + skin=skin, + description=description, ) raise RuntimeError("wait_for_approval can only be called inside a @workflow") @@ -3653,6 +3668,8 @@ async def _run_workflow_async(func, checkpoint: dict, input_args: dict): "key": info["key"], "timeout": info.get("timeout"), "form": info.get("form"), + "skin": info.get("skin"), + "description": info.get("description"), } if mode == "sleep": return { diff --git a/system_prompts/auto-generated/flow.md b/system_prompts/auto-generated/flow.md index 47891f0e07..d24db93d53 100644 --- a/system_prompts/auto-generated/flow.md +++ b/system_prompts/auto-generated/flow.md @@ -408,4 +408,4 @@ Reference a specific resource using `$res:` prefix: ## OpenFlow Schema -{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"Address of the account the flow runs on behalf of. Derived from on_behalf_of on read; accepted on write, where it is resolved to the account it names."},"on_behalf_of":{"type":"string","description":"The flow runs with the permissions of this identity: u/{username}, g/{group}, or a bare email when the username is itself email-shaped. The only stored half of the identity; on_behalf_of_email is derived from it. Omit it when writing and it is resolved from that address instead."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \"yellow\", \"#ffff00\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor"}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryAuto":{"type":"object","description":"Automatic context management","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","description":"Explicit message history","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Conversation memory configuration","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume","dbt"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'."},"description":{"type":"string","description":"Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script."},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\nRange: 1-1000.\n"}},"required":["user_message"]},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default `flow`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"agent":{"type":"string","description":"Path of a reusable `ai_agent` resource (hybrid linking). When set, the agent brain\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\nthat resource; the module's input_transforms then only carry the flow-local inputs\n(user_message/user_attachments).\n"},"tool_inputs":{"type":"object","description":"Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\nshared resource; overlaid onto the tools' input_transforms at runtime \u2014 including when\n`agent` is unset, since a step forked for editing keeps these overrides until it is saved\nback or unlinked.\n","additionalProperties":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}}},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}} \ No newline at end of file +{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"Address of the account the flow runs on behalf of. Derived from on_behalf_of on read; accepted on write, where it is resolved to the account it names."},"on_behalf_of":{"type":"string","description":"The flow runs with the permissions of this identity: u/{username}, g/{group}, or a bare email when the username is itself email-shaped. The only stored half of the identity; on_behalf_of_email is derived from it. Omit it when writing and it is resolved from that address instead."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \"yellow\", \"#ffff00\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor"}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"},"skin":{"type":"string","enum":["detailed","minimal"],"description":"How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryAuto":{"type":"object","description":"Automatic context management","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","description":"Explicit message history","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Conversation memory configuration","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume","dbt"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'."},"description":{"type":"string","description":"Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script."},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\nRange: 1-1000.\n"}},"required":["user_message"]},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default `flow`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"agent":{"type":"string","description":"Path of a reusable `ai_agent` resource (hybrid linking). When set, the agent brain\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\nthat resource; the module's input_transforms then only carry the flow-local inputs\n(user_message/user_attachments).\n"},"tool_inputs":{"type":"object","description":"Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\nshared resource; overlaid onto the tools' input_transforms at runtime \u2014 including when\n`agent` is unset, since a step forked for editing keeps these overrides until it is saved\nback or unlinked.\n","additionalProperties":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}}},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}} \ No newline at end of file diff --git a/system_prompts/auto-generated/prompts.ts b/system_prompts/auto-generated/prompts.ts index e13986771c..dc27234d60 100644 --- a/system_prompts/auto-generated/prompts.ts +++ b/system_prompts/auto-generated/prompts.ts @@ -1772,12 +1772,16 @@ workflow(fn: (...args: any[]) => Promise): void * resume exactly this approval — route them through your own channel. Without a * key the steps are named \`approval\`, \`approval_2\`, ... * + * \`skin: "minimal"\` shows approvers only the request (form and approve/reject) + * instead of the detailed page with the workflow's details. \`description\` is + * shown above the form: a string, or a rich value such as \`{ markdown: "..." }\`. + * * @example * const urls = await step("urls", () => getApprovalUrls("manager")); * await step("notify", () => sendEmail(urls.resume, urls.cancel)); * const { value, approver } = await waitForApproval({ key: "manager", timeout: 3600 }); */ -waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; }): PromiseLike<{ value: any; approver: string; approved: boolean }> +waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; skin?: "detailed" | "minimal"; description?: string | object; }): PromiseLike<{ value: any; approver: string; approved: boolean }> /** * Resume/cancel/approval-page URLs bound to one \`waitForApproval\` step. @@ -2646,13 +2650,17 @@ async def sleep(seconds: int) # form: Optional form schema for the approval page. # self_approval: Whether the user who triggered the flow can approve it (default True). # key: Optional checkpoint key naming this approval step. +# skin: \`\`"minimal"\`\` shows approvers only the request (form and approve/reject) +# instead of the detailed page with the workflow's details. +# description: Shown to approvers above the form: a string, or a rich value such as +# \`\`{"markdown": "..."}\`\`. # # Example:: # # urls = await step("urls", lambda: get_approval_urls("manager")) # await step("notify", lambda: send_email(urls["resume"], urls["cancel"])) # result = await wait_for_approval(key="manager", timeout=3600) -async def wait_for_approval(timeout: int = 1800, form: dict | None = None, self_approval: bool = True, key: str | None = None) -> dict +async def wait_for_approval(timeout: int = 1800, form: dict | None = None, self_approval: bool = True, key: str | None = None, skin: Literal['detailed', 'minimal'] | None = None, description: str | dict | None = None) -> dict # Process items in parallel with optional concurrency control. # @@ -2798,12 +2806,16 @@ export async function sleep(seconds: number): Promise * resume exactly this approval — route them through your own channel. Without a * key the steps are named \`approval\`, \`approval_2\`, ... * + * \`skin: "minimal"\` shows approvers only the request (form and approve/reject) + * instead of the detailed page with the workflow's details. \`description\` is + * shown above the form: a string, or a rich value such as \`{ markdown: "..." }\`. + * * @example * const urls = await step("urls", () => getApprovalUrls("manager")); * await step("notify", () => sendEmail(urls.resume, urls.cancel)); * const { value, approver } = await waitForApproval({ key: "manager", timeout: 3600 }); */ -export function waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; }): PromiseLike<{ value: any; approver: string; approved: boolean }> +export function waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; skin?: "detailed" | "minimal"; description?: string | object; }): PromiseLike<{ value: any; approver: string; approved: boolean }> /** * Resume/cancel/approval-page URLs bound to one \`waitForApproval\` step. @@ -2979,13 +2991,17 @@ async def sleep(seconds: int) # form: Optional form schema for the approval page. # self_approval: Whether the user who triggered the flow can approve it (default True). # key: Optional checkpoint key naming this approval step. +# skin: \`\`"minimal"\`\` shows approvers only the request (form and approve/reject) +# instead of the detailed page with the workflow's details. +# description: Shown to approvers above the form: a string, or a rich value such as +# \`\`{"markdown": "..."}\`\`. # # Example:: # # urls = await step("urls", lambda: get_approval_urls("manager")) # await step("notify", lambda: send_email(urls["resume"], urls["cancel"])) # result = await wait_for_approval(key="manager", timeout=3600) -async def wait_for_approval(timeout: int = 1800, form: dict | None = None, self_approval: bool = True, key: str | None = None) -> dict +async def wait_for_approval(timeout: int = 1800, form: dict | None = None, self_approval: bool = True, key: str | None = None, skin: Literal['detailed', 'minimal'] | None = None, description: str | dict | None = None) -> dict # Get the resume/cancel/approval-page URLs bound to one \`\`wait_for_approval\`\` step. # @@ -3176,7 +3192,7 @@ class SqlQuery: export const OPENFLOW_SCHEMA = `## OpenFlow Schema -{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"Address of the account the flow runs on behalf of. Derived from on_behalf_of on read; accepted on write, where it is resolved to the account it names."},"on_behalf_of":{"type":"string","description":"The flow runs with the permissions of this identity: u/{username}, g/{group}, or a bare email when the username is itself email-shaped. The only stored half of the identity; on_behalf_of_email is derived from it. Omit it when writing and it is resolved from that address instead."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \\"yellow\\", \\"#ffff00\\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \\u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor"}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryAuto":{"type":"object","description":"Automatic context management","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","description":"Explicit message history","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Conversation memory configuration","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume","dbt"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'."},"description":{"type":"string","description":"Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script."},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\\nValid values: 'text' (default) - plain text response, 'image' - image generation\\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\\nFormat: Array<{ bucket: string, key: string }> - S3 object references\\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\\nRange: 0.0 to 2.0 (provider-dependent)\\n- 0.0 = deterministic, focused responses\\n- 0.7 = balanced (common default)\\n- 1.0+ = more creative/random\\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\\nRange: 1-1000.\\n"}},"required":["user_message"]},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default \`flow\`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"agent":{"type":"string","description":"Path of a reusable \`ai_agent\` resource (hybrid linking). When set, the agent brain\\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\\nthat resource; the module's input_transforms then only carry the flow-local inputs\\n(user_message/user_attachments).\\n"},"tool_inputs":{"type":"object","description":"Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\\nshared resource; overlaid onto the tools' input_transforms at runtime \\u2014 including when\\n\`agent\` is unset, since a step forked for editing keeps these overrides until it is saved\\nback or unlinked.\\n","additionalProperties":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}}},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}}`; +{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"Address of the account the flow runs on behalf of. Derived from on_behalf_of on read; accepted on write, where it is resolved to the account it names."},"on_behalf_of":{"type":"string","description":"The flow runs with the permissions of this identity: u/{username}, g/{group}, or a bare email when the username is itself email-shaped. The only stored half of the identity; on_behalf_of_email is derived from it. Omit it when writing and it is resolved from that address instead."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \\"yellow\\", \\"#ffff00\\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \\u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor"}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"},"skin":{"type":"string","enum":["detailed","minimal"],"description":"How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryAuto":{"type":"object","description":"Automatic context management","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","description":"Explicit message history","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Conversation memory configuration","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume","dbt"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'."},"description":{"type":"string","description":"Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script."},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\\nValid values: 'text' (default) - plain text response, 'image' - image generation\\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\\nFormat: Array<{ bucket: string, key: string }> - S3 object references\\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\\nRange: 0.0 to 2.0 (provider-dependent)\\n- 0.0 = deterministic, focused responses\\n- 0.7 = balanced (common default)\\n- 1.0+ = more creative/random\\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\\nRange: 1-1000.\\n"}},"required":["user_message"]},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default \`flow\`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"agent":{"type":"string","description":"Path of a reusable \`ai_agent\` resource (hybrid linking). When set, the agent brain\\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\\nthat resource; the module's input_transforms then only carry the flow-local inputs\\n(user_message/user_attachments).\\n"},"tool_inputs":{"type":"object","description":"Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\\nshared resource; overlaid onto the tools' input_transforms at runtime \\u2014 including when\\n\`agent\` is unset, since a step forked for editing keeps these overrides until it is saved\\nback or unlinked.\\n","additionalProperties":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}}},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}}`; export const CLI_COMMANDS = `# Windmill CLI Commands diff --git a/system_prompts/auto-generated/script.md b/system_prompts/auto-generated/script.md index 4d7c02a90d..7883a8ba4b 100644 --- a/system_prompts/auto-generated/script.md +++ b/system_prompts/auto-generated/script.md @@ -1927,12 +1927,16 @@ workflow(fn: (...args: any[]) => Promise): void * resume exactly this approval — route them through your own channel. Without a * key the steps are named `approval`, `approval_2`, ... * + * `skin: "minimal"` shows approvers only the request (form and approve/reject) + * instead of the detailed page with the workflow's details. `description` is + * shown above the form: a string, or a rich value such as `{ markdown: "..." }`. + * * @example * const urls = await step("urls", () => getApprovalUrls("manager")); * await step("notify", () => sendEmail(urls.resume, urls.cancel)); * const { value, approver } = await waitForApproval({ key: "manager", timeout: 3600 }); */ -waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; }): PromiseLike<{ value: any; approver: string; approved: boolean }> +waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; skin?: "detailed" | "minimal"; description?: string | object; }): PromiseLike<{ value: any; approver: string; approved: boolean }> /** * Resume/cancel/approval-page URLs bound to one `waitForApproval` step. @@ -2801,13 +2805,17 @@ async def sleep(seconds: int) # form: Optional form schema for the approval page. # self_approval: Whether the user who triggered the flow can approve it (default True). # key: Optional checkpoint key naming this approval step. +# skin: ``"minimal"`` shows approvers only the request (form and approve/reject) +# instead of the detailed page with the workflow's details. +# description: Shown to approvers above the form: a string, or a rich value such as +# ``{"markdown": "..."}``. # # Example:: # # urls = await step("urls", lambda: get_approval_urls("manager")) # await step("notify", lambda: send_email(urls["resume"], urls["cancel"])) # result = await wait_for_approval(key="manager", timeout=3600) -async def wait_for_approval(timeout: int = 1800, form: dict | None = None, self_approval: bool = True, key: str | None = None) -> dict +async def wait_for_approval(timeout: int = 1800, form: dict | None = None, self_approval: bool = True, key: str | None = None, skin: Literal['detailed', 'minimal'] | None = None, description: str | dict | None = None) -> dict # Process items in parallel with optional concurrency control. # diff --git a/system_prompts/auto-generated/sdks/python.md b/system_prompts/auto-generated/sdks/python.md index a060267f35..4aacf4d1b4 100644 --- a/system_prompts/auto-generated/sdks/python.md +++ b/system_prompts/auto-generated/sdks/python.md @@ -749,13 +749,17 @@ async def sleep(seconds: int) # form: Optional form schema for the approval page. # self_approval: Whether the user who triggered the flow can approve it (default True). # key: Optional checkpoint key naming this approval step. +# skin: ``"minimal"`` shows approvers only the request (form and approve/reject) +# instead of the detailed page with the workflow's details. +# description: Shown to approvers above the form: a string, or a rich value such as +# ``{"markdown": "..."}``. # # Example:: # # urls = await step("urls", lambda: get_approval_urls("manager")) # await step("notify", lambda: send_email(urls["resume"], urls["cancel"])) # result = await wait_for_approval(key="manager", timeout=3600) -async def wait_for_approval(timeout: int = 1800, form: dict | None = None, self_approval: bool = True, key: str | None = None) -> dict +async def wait_for_approval(timeout: int = 1800, form: dict | None = None, self_approval: bool = True, key: str | None = None, skin: Literal['detailed', 'minimal'] | None = None, description: str | dict | None = None) -> dict # Process items in parallel with optional concurrency control. # diff --git a/system_prompts/auto-generated/sdks/typescript.md b/system_prompts/auto-generated/sdks/typescript.md index 1269a3e497..b764b38a9e 100644 --- a/system_prompts/auto-generated/sdks/typescript.md +++ b/system_prompts/auto-generated/sdks/typescript.md @@ -492,12 +492,16 @@ workflow(fn: (...args: any[]) => Promise): void * resume exactly this approval — route them through your own channel. Without a * key the steps are named `approval`, `approval_2`, ... * + * `skin: "minimal"` shows approvers only the request (form and approve/reject) + * instead of the detailed page with the workflow's details. `description` is + * shown above the form: a string, or a rich value such as `{ markdown: "..." }`. + * * @example * const urls = await step("urls", () => getApprovalUrls("manager")); * await step("notify", () => sendEmail(urls.resume, urls.cancel)); * const { value, approver } = await waitForApproval({ key: "manager", timeout: 3600 }); */ -waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; }): PromiseLike<{ value: any; approver: string; approved: boolean }> +waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; skin?: "detailed" | "minimal"; description?: string | object; }): PromiseLike<{ value: any; approver: string; approved: boolean }> /** * Resume/cancel/approval-page URLs bound to one `waitForApproval` step. diff --git a/system_prompts/auto-generated/sdks/wac-python.md b/system_prompts/auto-generated/sdks/wac-python.md index e7f92d32d0..816ea4959b 100644 --- a/system_prompts/auto-generated/sdks/wac-python.md +++ b/system_prompts/auto-generated/sdks/wac-python.md @@ -135,13 +135,17 @@ async def sleep(seconds: int) # form: Optional form schema for the approval page. # self_approval: Whether the user who triggered the flow can approve it (default True). # key: Optional checkpoint key naming this approval step. +# skin: ``"minimal"`` shows approvers only the request (form and approve/reject) +# instead of the detailed page with the workflow's details. +# description: Shown to approvers above the form: a string, or a rich value such as +# ``{"markdown": "..."}``. # # Example:: # # urls = await step("urls", lambda: get_approval_urls("manager")) # await step("notify", lambda: send_email(urls["resume"], urls["cancel"])) # result = await wait_for_approval(key="manager", timeout=3600) -async def wait_for_approval(timeout: int = 1800, form: dict | None = None, self_approval: bool = True, key: str | None = None) -> dict +async def wait_for_approval(timeout: int = 1800, form: dict | None = None, self_approval: bool = True, key: str | None = None, skin: Literal['detailed', 'minimal'] | None = None, description: str | dict | None = None) -> dict # Get the resume/cancel/approval-page URLs bound to one ``wait_for_approval`` step. # diff --git a/system_prompts/auto-generated/sdks/wac-typescript.md b/system_prompts/auto-generated/sdks/wac-typescript.md index afaeac369a..66eecfe761 100644 --- a/system_prompts/auto-generated/sdks/wac-typescript.md +++ b/system_prompts/auto-generated/sdks/wac-typescript.md @@ -117,12 +117,16 @@ export async function sleep(seconds: number): Promise * resume exactly this approval — route them through your own channel. Without a * key the steps are named `approval`, `approval_2`, ... * + * `skin: "minimal"` shows approvers only the request (form and approve/reject) + * instead of the detailed page with the workflow's details. `description` is + * shown above the form: a string, or a rich value such as `{ markdown: "..." }`. + * * @example * const urls = await step("urls", () => getApprovalUrls("manager")); * await step("notify", () => sendEmail(urls.resume, urls.cancel)); * const { value, approver } = await waitForApproval({ key: "manager", timeout: 3600 }); */ -export function waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; }): PromiseLike<{ value: any; approver: string; approved: boolean }> +export function waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; skin?: "detailed" | "minimal"; description?: string | object; }): PromiseLike<{ value: any; approver: string; approved: boolean }> /** * Resume/cancel/approval-page URLs bound to one `waitForApproval` step. diff --git a/system_prompts/auto-generated/skills/write-flow/SKILL.md b/system_prompts/auto-generated/skills/write-flow/SKILL.md index 925ffbe6ec..8c7b7bfec5 100644 --- a/system_prompts/auto-generated/skills/write-flow/SKILL.md +++ b/system_prompts/auto-generated/skills/write-flow/SKILL.md @@ -496,4 +496,4 @@ Reference a specific resource using `$res:` prefix: ## OpenFlow Schema -{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"Address of the account the flow runs on behalf of. Derived from on_behalf_of on read; accepted on write, where it is resolved to the account it names."},"on_behalf_of":{"type":"string","description":"The flow runs with the permissions of this identity: u/{username}, g/{group}, or a bare email when the username is itself email-shaped. The only stored half of the identity; on_behalf_of_email is derived from it. Omit it when writing and it is resolved from that address instead."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \"yellow\", \"#ffff00\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor"}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryAuto":{"type":"object","description":"Automatic context management","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","description":"Explicit message history","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Conversation memory configuration","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume","dbt"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'."},"description":{"type":"string","description":"Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script."},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\nRange: 1-1000.\n"}},"required":["user_message"]},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default `flow`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"agent":{"type":"string","description":"Path of a reusable `ai_agent` resource (hybrid linking). When set, the agent brain\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\nthat resource; the module's input_transforms then only carry the flow-local inputs\n(user_message/user_attachments).\n"},"tool_inputs":{"type":"object","description":"Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\nshared resource; overlaid onto the tools' input_transforms at runtime \u2014 including when\n`agent` is unset, since a step forked for editing keeps these overrides until it is saved\nback or unlinked.\n","additionalProperties":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}}},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}} \ No newline at end of file +{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"Address of the account the flow runs on behalf of. Derived from on_behalf_of on read; accepted on write, where it is resolved to the account it names."},"on_behalf_of":{"type":"string","description":"The flow runs with the permissions of this identity: u/{username}, g/{group}, or a bare email when the username is itself email-shaped. The only stored half of the identity; on_behalf_of_email is derived from it. Omit it when writing and it is resolved from that address instead."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \"yellow\", \"#ffff00\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor"}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"},"skin":{"type":"string","enum":["detailed","minimal"],"description":"How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryAuto":{"type":"object","description":"Automatic context management","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","description":"Explicit message history","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Conversation memory configuration","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume","dbt"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'."},"description":{"type":"string","description":"Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script."},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\nRange: 1-1000.\n"}},"required":["user_message"]},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default `flow`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"agent":{"type":"string","description":"Path of a reusable `ai_agent` resource (hybrid linking). When set, the agent brain\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\nthat resource; the module's input_transforms then only carry the flow-local inputs\n(user_message/user_attachments).\n"},"tool_inputs":{"type":"object","description":"Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\nshared resource; overlaid onto the tools' input_transforms at runtime \u2014 including when\n`agent` is unset, since a step forked for editing keeps these overrides until it is saved\nback or unlinked.\n","additionalProperties":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}}},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}} \ No newline at end of file diff --git a/system_prompts/auto-generated/skills/write-script-bun/SKILL.md b/system_prompts/auto-generated/skills/write-script-bun/SKILL.md index 74ef8f1908..c89a87fa5d 100644 --- a/system_prompts/auto-generated/skills/write-script-bun/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-bun/SKILL.md @@ -663,12 +663,16 @@ workflow(fn: (...args: any[]) => Promise): void * resume exactly this approval — route them through your own channel. Without a * key the steps are named `approval`, `approval_2`, ... * + * `skin: "minimal"` shows approvers only the request (form and approve/reject) + * instead of the detailed page with the workflow's details. `description` is + * shown above the form: a string, or a rich value such as `{ markdown: "..." }`. + * * @example * const urls = await step("urls", () => getApprovalUrls("manager")); * await step("notify", () => sendEmail(urls.resume, urls.cancel)); * const { value, approver } = await waitForApproval({ key: "manager", timeout: 3600 }); */ -waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; }): PromiseLike<{ value: any; approver: string; approved: boolean }> +waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; skin?: "detailed" | "minimal"; description?: string | object; }): PromiseLike<{ value: any; approver: string; approved: boolean }> /** * Resume/cancel/approval-page URLs bound to one `waitForApproval` step. diff --git a/system_prompts/auto-generated/skills/write-script-bunnative/SKILL.md b/system_prompts/auto-generated/skills/write-script-bunnative/SKILL.md index 0c7b6aecde..82911c1096 100644 --- a/system_prompts/auto-generated/skills/write-script-bunnative/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-bunnative/SKILL.md @@ -663,12 +663,16 @@ workflow(fn: (...args: any[]) => Promise): void * resume exactly this approval — route them through your own channel. Without a * key the steps are named `approval`, `approval_2`, ... * + * `skin: "minimal"` shows approvers only the request (form and approve/reject) + * instead of the detailed page with the workflow's details. `description` is + * shown above the form: a string, or a rich value such as `{ markdown: "..." }`. + * * @example * const urls = await step("urls", () => getApprovalUrls("manager")); * await step("notify", () => sendEmail(urls.resume, urls.cancel)); * const { value, approver } = await waitForApproval({ key: "manager", timeout: 3600 }); */ -waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; }): PromiseLike<{ value: any; approver: string; approved: boolean }> +waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; skin?: "detailed" | "minimal"; description?: string | object; }): PromiseLike<{ value: any; approver: string; approved: boolean }> /** * Resume/cancel/approval-page URLs bound to one `waitForApproval` step. diff --git a/system_prompts/auto-generated/skills/write-script-deno/SKILL.md b/system_prompts/auto-generated/skills/write-script-deno/SKILL.md index 930f078a58..95f528223d 100644 --- a/system_prompts/auto-generated/skills/write-script-deno/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-deno/SKILL.md @@ -665,12 +665,16 @@ workflow(fn: (...args: any[]) => Promise): void * resume exactly this approval — route them through your own channel. Without a * key the steps are named `approval`, `approval_2`, ... * + * `skin: "minimal"` shows approvers only the request (form and approve/reject) + * instead of the detailed page with the workflow's details. `description` is + * shown above the form: a string, or a rich value such as `{ markdown: "..." }`. + * * @example * const urls = await step("urls", () => getApprovalUrls("manager")); * await step("notify", () => sendEmail(urls.resume, urls.cancel)); * const { value, approver } = await waitForApproval({ key: "manager", timeout: 3600 }); */ -waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; }): PromiseLike<{ value: any; approver: string; approved: boolean }> +waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; skin?: "detailed" | "minimal"; description?: string | object; }): PromiseLike<{ value: any; approver: string; approved: boolean }> /** * Resume/cancel/approval-page URLs bound to one `waitForApproval` step. diff --git a/system_prompts/auto-generated/skills/write-script-python3/SKILL.md b/system_prompts/auto-generated/skills/write-script-python3/SKILL.md index b91c52d986..8100eebc25 100644 --- a/system_prompts/auto-generated/skills/write-script-python3/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-python3/SKILL.md @@ -934,13 +934,17 @@ async def sleep(seconds: int) # form: Optional form schema for the approval page. # self_approval: Whether the user who triggered the flow can approve it (default True). # key: Optional checkpoint key naming this approval step. +# skin: ``"minimal"`` shows approvers only the request (form and approve/reject) +# instead of the detailed page with the workflow's details. +# description: Shown to approvers above the form: a string, or a rich value such as +# ``{"markdown": "..."}``. # # Example:: # # urls = await step("urls", lambda: get_approval_urls("manager")) # await step("notify", lambda: send_email(urls["resume"], urls["cancel"])) # result = await wait_for_approval(key="manager", timeout=3600) -async def wait_for_approval(timeout: int = 1800, form: dict | None = None, self_approval: bool = True, key: str | None = None) -> dict +async def wait_for_approval(timeout: int = 1800, form: dict | None = None, self_approval: bool = True, key: str | None = None, skin: Literal['detailed', 'minimal'] | None = None, description: str | dict | None = None) -> dict # Process items in parallel with optional concurrency control. # diff --git a/system_prompts/auto-generated/skills/write-workflow-as-code/SKILL.md b/system_prompts/auto-generated/skills/write-workflow-as-code/SKILL.md index 9da6969e52..c41bd54f04 100644 --- a/system_prompts/auto-generated/skills/write-workflow-as-code/SKILL.md +++ b/system_prompts/auto-generated/skills/write-workflow-as-code/SKILL.md @@ -360,12 +360,16 @@ export async function sleep(seconds: number): Promise * resume exactly this approval — route them through your own channel. Without a * key the steps are named `approval`, `approval_2`, ... * + * `skin: "minimal"` shows approvers only the request (form and approve/reject) + * instead of the detailed page with the workflow's details. `description` is + * shown above the form: a string, or a rich value such as `{ markdown: "..." }`. + * * @example * const urls = await step("urls", () => getApprovalUrls("manager")); * await step("notify", () => sendEmail(urls.resume, urls.cancel)); * const { value, approver } = await waitForApproval({ key: "manager", timeout: 3600 }); */ -export function waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; }): PromiseLike<{ value: any; approver: string; approved: boolean }> +export function waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; skin?: "detailed" | "minimal"; description?: string | object; }): PromiseLike<{ value: any; approver: string; approved: boolean }> /** * Resume/cancel/approval-page URLs bound to one `waitForApproval` step. @@ -541,13 +545,17 @@ async def sleep(seconds: int) # form: Optional form schema for the approval page. # self_approval: Whether the user who triggered the flow can approve it (default True). # key: Optional checkpoint key naming this approval step. +# skin: ``"minimal"`` shows approvers only the request (form and approve/reject) +# instead of the detailed page with the workflow's details. +# description: Shown to approvers above the form: a string, or a rich value such as +# ``{"markdown": "..."}``. # # Example:: # # urls = await step("urls", lambda: get_approval_urls("manager")) # await step("notify", lambda: send_email(urls["resume"], urls["cancel"])) # result = await wait_for_approval(key="manager", timeout=3600) -async def wait_for_approval(timeout: int = 1800, form: dict | None = None, self_approval: bool = True, key: str | None = None) -> dict +async def wait_for_approval(timeout: int = 1800, form: dict | None = None, self_approval: bool = True, key: str | None = None, skin: Literal['detailed', 'minimal'] | None = None, description: str | dict | None = None) -> dict # Get the resume/cancel/approval-page URLs bound to one ``wait_for_approval`` step. # diff --git a/typescript-client/client.ts b/typescript-client/client.ts index 23ca08786c..7b10c732be 100644 --- a/typescript-client/client.ts +++ b/typescript-client/client.ts @@ -1962,6 +1962,8 @@ export class WorkflowCtx { form?: object; selfApproval?: boolean; key?: string; + skin?: "detailed" | "minimal"; + description?: string | object; }): PromiseLike<{ value: any; approver: string; approved: boolean }> { this._rethrowSwallowed(); if (options?.key !== undefined) assertUsableStepKey(options.key, "waitForApproval key"); @@ -1996,6 +1998,8 @@ export class WorkflowCtx { timeout: options?.timeout ?? 1800, form: options?.form, self_approval_disabled: !(options?.selfApproval ?? true), + skin: options?.skin, + description: options?.description, steps: [], }); } @@ -2459,6 +2463,10 @@ export function workflow(fn: (...args: any[]) => Promise) { * resume exactly this approval — route them through your own channel. Without a * key the steps are named `approval`, `approval_2`, ... * + * `skin: "minimal"` shows approvers only the request (form and approve/reject) + * instead of the detailed page with the workflow's details. `description` is + * shown above the form: a string, or a rich value such as `{ markdown: "..." }`. + * * @example * const urls = await step("urls", () => getApprovalUrls("manager")); * await step("notify", () => sendEmail(urls.resume, urls.cancel)); @@ -2469,6 +2477,8 @@ export function waitForApproval(options?: { form?: object; selfApproval?: boolean; key?: string; + skin?: "detailed" | "minimal"; + description?: string | object; }): PromiseLike<{ value: any; approver: string; approved: boolean }> { const ctx: WorkflowCtx | null = _workflowCtx ?? Reflect.get(globalThis, "__wmill_wf_ctx"); if (!ctx) { From d87f089288996af9ea7e3b017a5ef35d4ded880d Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 10 Sep 2026 20:57:50 +0200 Subject: [PATCH 068/144] feat: tuck other users' spaces into a collapsible home tree row (#11073) * feat: group other users' spaces under a collapsible row in the home tree Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_013eAE5cpKPdtH9DNJm5TjcH * fix: page the other users group on its own and use a design-system toggle Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_013eAE5cpKPdtH9DNJm5TjcH --------- Co-authored-by: Claude Opus 5 (1M context) --- .../src/lib/components/home/ItemsList.svelte | 1 + .../src/lib/components/home/TreeView.svelte | 25 ++- .../lib/components/home/TreeViewRoot.svelte | 188 +++++++++++++++--- 3 files changed, 174 insertions(+), 40 deletions(-) diff --git a/frontend/src/lib/components/home/ItemsList.svelte b/frontend/src/lib/components/home/ItemsList.svelte index 95f95bf6ce..35b1c5a6f3 100644 --- a/frontend/src/lib/components/home/ItemsList.svelte +++ b/frontend/src/lib/components/home/ItemsList.svelte @@ -1991,6 +1991,7 @@ allUsers={treeInjectUsers} ownerCounts={!searching && labelFilter == undefined ? ownerCounts : undefined} selfUsername={$userStore?.username} + groupOtherUsers={treeLazyMode} ownerLoad={treeLazyMode ? ownerLoad : undefined} onExpandOwner={treeLazyMode ? loadOwnerItems : undefined} onCollapseOwner={treeLazyMode ? collapseOwner : undefined} diff --git a/frontend/src/lib/components/home/TreeView.svelte b/frontend/src/lib/components/home/TreeView.svelte index 31eae20b16..0581700316 100644 --- a/frontend/src/lib/components/home/TreeView.svelte +++ b/frontend/src/lib/components/home/TreeView.svelte @@ -32,8 +32,9 @@ // `all` pages the prefix to the end in one call instead of fetching a single page. onExpandOwner?: (prefix: string, more?: boolean, opts?: { all?: boolean }) => void onCollapseOwner?: (prefix: string) => void - // Position of this node among the rendered root nodes; "expand all" only - // auto-loads the first EXPAND_ALL_LOAD_LIMIT of them (see the effect below). + // This root owner's place in line for "expand all", which only auto-loads the first + // EXPAND_ALL_LOAD_LIMIT (see the effect below). Not always its rendered position: + // owners nested under a grouping row are ranked after the rest. rootIndex?: number showEditButton?: boolean // Path prefix of the parent node, so this one can name its own (`ownerLoad` and @@ -43,6 +44,10 @@ // is grouped under this node is only part of it: counts render as "N+" and the // node offers to load the rest of itself. ancestorHasMore?: boolean + // Visual nesting on top of `depth`. `depth` stays semantic (0 is a top-level owner + // that loads lazily), so an owner shown inside a grouping row is indented through + // this rather than by raising its depth. + indent?: number } let { @@ -59,9 +64,12 @@ rootIndex = 0, showEditButton = true, parentPrefix, - ancestorHasMore = false + ancestorHasMore = false, + indent = 0 }: Props = $props() + let visualDepth = $derived(depth + indent) + // Bounds the request burst from "expand all": however many root owners the tree // renders (its slice grows as you scroll), it fetches at most this many. Lazy owners // past the cap stay collapsed and load on a single click (see the effect). @@ -256,7 +264,7 @@ >
    0 ? `padding-left: ${depth * 16}px;` : ''} + style={visualDepth > 0 ? `padding-left: ${visualDepth * 16}px;` : ''} >
    {#if isUser(item)} @@ -310,7 +318,7 @@ Pipeline @@ -335,12 +343,13 @@ {showCode} {showEditButton} depth={depth + 1} + {indent} /> {/each} {#if effectiveMax < item.items.length}
    @@ -377,7 +386,7 @@ as rows still missing. -->
    Showing {loadedHere}{ownerTotal != undefined ? ` of ${ownerTotal}` : ''} items in {nodePrefix} @@ -427,6 +436,6 @@ on:appChanged on:rawAppChanged on:reload - {depth} + depth={visualDepth} /> {/if} diff --git a/frontend/src/lib/components/home/TreeViewRoot.svelte b/frontend/src/lib/components/home/TreeViewRoot.svelte index 39c2188e59..062dfb69f8 100644 --- a/frontend/src/lib/components/home/TreeViewRoot.svelte +++ b/frontend/src/lib/components/home/TreeViewRoot.svelte @@ -1,8 +1,10 @@ +{#snippet ownerNode(node: RootNode, loadRank: number, indent: number)} + +{/snippet} + {#if groupedItems === 'loading'}
    {:else}
    - {#each groupedItems.slice(0, nbDisplayed) as item, rootIndex ('folderName' in item ? `f__${item.folderName}` : 'username' in item ? `u__${item.username}` : `i__${item.type}__${item.path}`)} - {#if item} - + {#each rows.slice(0, nbDisplayed) as row (row.kind === 'otherUsers' ? 'other_users' : 'folderName' in row.node ? `f__${row.node.folderName}` : 'username' in row.node ? `u__${row.node.username}` : `i__${row.node.type}__${row.node.path}`)} + {#if row.kind === 'otherUsers'} + + + +
    +
    + +
    + Other users +
    + ({pluralize(otherUsers.length, 'user')}{otherUsersItemCount != undefined + ? ` · ${pluralize(otherUsersItemCount, 'item')}` + : ''}) +
    +
    +
    +
    + {#if otherUsersOpen} + + {#each otherUsers.slice(0, nbOtherUsersDisplayed) as user, i (user.username)} + {@render ownerNode(user, ownerRowCount + i, 1)} + {/each} + {#if nbOtherUsersDisplayed < otherUsers.length} +
    + + Showing {nbOtherUsersDisplayed} of {otherUsers.length} users + + +
    + {/if} + {/if} + {:else} + {@render ownerNode(row.node, row.loadRank, 0)} {/if} {/each} - {#if nbDisplayed < groupedItems.length || hasMoreServer} + {#if nbDisplayed < rows.length || hasMoreServer}
    - {#if nbDisplayed < groupedItems.length} - Showing {nbDisplayed} of {groupedItems.length} folders and users + {#if nbDisplayed < rows.length} + Showing {shownOwnerRowCount} of {ownerRowCount} folders and users {:else} @@ -231,12 +355,12 @@ unifiedSize="sm" variant="subtle" on:click={() => { - if (nbDisplayed < groupedItems.length) - nbDisplayed = Math.min(nbDisplayed + ROOT_PAGE, groupedItems.length) + if (nbDisplayed < rows.length) + nbDisplayed = Math.min(nbDisplayed + ROOT_PAGE, rows.length) else onLoadMore?.() }} > - {nbDisplayed < groupedItems.length ? 'Show more' : 'Load more'} + {nbDisplayed < rows.length ? 'Show more' : 'Load more'}
    {/if} From 8ecbd339eef7314a93d599fcea4377299d4c493d Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 10 Sep 2026 22:45:34 +0200 Subject: [PATCH 069/144] fix: skip the deploy PR when the git sync push committed nothing (#11076) Claude-Session: https://claude.ai/code/session_01WVDyzszjbvSssMN6HpnZZw Co-authored-by: Claude Opus 5 (1M context) --- backend/windmill-common/src/workspaces.rs | 2 +- integration_tests/test/git_sync_test.py | 63 +++++++++++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index ef013fc2a8..45c8695585 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -183,7 +183,7 @@ pub enum ObjectType { DatatableMigration, } -pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28949/sync-script-to-git-repo-windmill"; +pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28958/sync-script-to-git-repo-windmill"; /// Hub script that applies a repository's state back into a workspace /// (the repo → Windmill / "pull" direction). Same script the UI runs from diff --git a/integration_tests/test/git_sync_test.py b/integration_tests/test/git_sync_test.py index 47ac02351e..3f761e36f5 100644 --- a/integration_tests/test/git_sync_test.py +++ b/integration_tests/test/git_sync_test.py @@ -771,6 +771,69 @@ class TestGitSync(GitSyncTestBase): f"Expected folder-grouped branch name containing '{expected_folder_part}', got: {deploy_branch}", ) + def _new_sync_job_results(self, seen: set, path: str) -> list: + """Results of the deploy callbacks not in `seen` that pushed `path`.""" + results = [] + for listed in self._client.get_completed_jobs(job_kinds="deploymentcallback"): + if listed["id"] in seen: + continue + job = self._client._client.get( + f"/api/w/{self._client._workspace}/jobs_u/get/{listed['id']}" + ).json() + items = (job.get("args") or {}).get("items") or [] + if any(item.get("path") == path for item in items): + results.append(job.get("result")) + return results + + def test_promotion_push_reports_whether_it_pushed(self): + """The push job's result says whether a commit was pushed, and the + PR-on-deploy hook relies on it: on `pushed: false` it skips, otherwise + it asks the host for a PR on the deploy branch, which for a push that + committed nothing was never created (GitHub: 422 "head invalid").""" + repo_name, _ = self._create_test_repo() + resource_path = self._setup_git_sync_resource(repo_name) + + # Saving the config is itself a settings deploy, and it commits nothing: + # the repo has no wmill.yaml including settings, so none are pulled. + seen = {j["id"] for j in self._client.get_completed_jobs(job_kinds="deploymentcallback")} + self._configure_single_repo_sync( + resource_path, + include_type=["script", "settings"], + use_individual_branch=True, + group_by_folder=True, + ) + self._wait_until( + lambda: self._new_sync_job_results(seen, "settings.yaml"), + timeout=90, + message="no push job for the settings deploy", + ) + results = self._new_sync_job_results(seen, "settings.yaml") + self.assertEqual( + [(r or {}).get("pushed") for r in results], + [False], + f"a settings deploy that committed nothing must report pushed: false, got: {results}", + ) + + folder_name = unique_name("pushed") + self._create_folder(folder_name) + seen = {j["id"] for j in self._client.get_completed_jobs(job_kinds="deploymentcallback")} + script_path = f"f/{folder_name}/{unique_name('script')}" + self._client.create_script( + path=script_path, + content=ts_script("return 'pushed'"), + language="bun", + ) + self._wait_until( + lambda: self._new_sync_job_results(seen, script_path), + timeout=90, + message=f"no push job for {script_path}", + ) + results = self._new_sync_job_results(seen, script_path) + self.assertTrue( + all((r or {}).get("pushed") is True for r in results), + f"a deploy that committed must report pushed: true, got: {results}", + ) + # ────────────────────────────────────────────────── # Exclude path filtering # ────────────────────────────────────────────────── From a9b0d871a40697a1412815ed12392cfc1b2f1dfc Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 10 Sep 2026 22:54:54 +0200 Subject: [PATCH 070/144] surface OTEL env vars in the settings page and startup log (#11074) * feat: surface OTEL env vars in the settings page and startup log Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01RbLzVfFDZ9pjBGHSzCSrNZ * chore: log OTEL compression and per-signal timeouts in the startup config Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01RbLzVfFDZ9pjBGHSzCSrNZ --------- Co-authored-by: Claude Opus 5 (1M context) --- .../windmill-common/src/global_settings.rs | 24 +++++++++++++++++++ .../src/lib/components/InstanceSetting.svelte | 8 +++++++ 2 files changed, 32 insertions(+) diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index fdeeb2a92d..1d590e38bf 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -433,6 +433,30 @@ pub const ENV_SETTINGS: &[&str] = &[ "OTEL_METRICS", "OTEL_TRACING", "OTEL_LOGS", + // The OTEL_EXPORTER_OTLP_*HEADERS variables are left out: they carry exporter API keys, and + // this list is logged at startup and returned to superadmins by `get_local_settings`. + "OTEL_METRICS_ENABLED", + "OTEL_TRACING_ENABLED", + "OTEL_LOGS_ENABLED", + "OTEL_EXPORTER_OTLP_ENDPOINT", + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", + "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", + "OTEL_EXPORTER_OTLP_LOGS_ENDPOINT", + "OTEL_EXPORTER_OTLP_PROTOCOL", + "OTEL_EXPORTER_OTLP_COMPRESSION", + "OTEL_EXPORTER_OTLP_TIMEOUT", + "OTEL_EXPORTER_OTLP_TRACES_TIMEOUT", + "OTEL_EXPORTER_OTLP_METRICS_TIMEOUT", + "OTEL_EXPORTER_OTLP_LOGS_TIMEOUT", + "OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE", + "OTEL_METRIC_EXPORT_INTERVAL", + "OTEL_SERVICE_NAME", + "OTEL_SERVICE_VERSION", + "OTEL_HOST_NAME", + "OTEL_ENVIRONMENT", + "OTEL_RESOURCE_ATTRIBUTES", + "OTEL_JOB_LOGS", + "OTEL_TRACES_RETENTION_SECS", "DISABLE_S3_STORE", "PG_SCHEMA", "PG_LISTENER_REFRESH_PERIOD_SECS", diff --git a/frontend/src/lib/components/InstanceSetting.svelte b/frontend/src/lib/components/InstanceSetting.svelte index 7b6c5e312e..1e67652f60 100644 --- a/frontend/src/lib/components/InstanceSetting.svelte +++ b/frontend/src/lib/components/InstanceSetting.svelte @@ -31,6 +31,7 @@ import IndexerJobIndexSettings from './instanceSettings/IndexerJobIndexSettings.svelte' import IndexerLogIndexSettings from './instanceSettings/IndexerLogIndexSettings.svelte' import TextInput from './text_input/TextInput.svelte' + import Description from './Description.svelte' import SettingCard from './instanceSettings/SettingCard.svelte' interface Props { @@ -694,6 +695,13 @@
    + + Service name, environment, resource attributes, metrics temporality and other + options are set with environment variables. + {/if}
    {:else if setting.fieldType == 'otel_tracing_proxy'} From 7915540f66a75c13c1a78cafee45104a0ab3d26c Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 10 Sep 2026 22:57:04 +0200 Subject: [PATCH 071/144] chore(main): release 1.809.0 (#11045) * chore(main): release 1.809.0 * Apply automatic changes --------- Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com> --- .release-please-manifest.json | 2 +- CHANGELOG.md | 35 ++ backend/Cargo.lock | 306 ++++++++++-------- backend/Cargo.toml | 4 +- .../parsers/windmill-parser-wasm/Cargo.lock | 48 +-- .../parsers/windmill-parser-wasm/Cargo.toml | 2 +- backend/windmill-api/openapi.yaml | 2 +- benchmarks/lib.ts | 2 +- cli/src/core/constants.ts | 2 +- frontend/package-lock.json | 4 +- frontend/package.json | 2 +- lsp/Pipfile | 2 +- openflow.openapi.yaml | 2 +- .../WindmillClient/WindmillClient.psd1 | 2 +- python-client/wmill/pyproject.toml | 2 +- typescript-client/jsr.json | 2 +- typescript-client/package.json | 2 +- version.txt | 2 +- windmill-yaml-validator/package-lock.json | 4 +- windmill-yaml-validator/package.json | 2 +- 20 files changed, 254 insertions(+), 175 deletions(-) diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 204044d7a0..4100cf6bbc 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "1.808.0" + ".": "1.809.0" } diff --git a/CHANGELOG.md b/CHANGELOG.md index 72174817f0..d82cc8a185 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,40 @@ # Changelog +## [1.809.0](https://github.com/windmill-labs/windmill/compare/v1.808.0...v1.809.0) (2026-09-10) + + +### Features + +* add a minimal skin for the approval page and slack/teams ([#11061](https://github.com/windmill-labs/windmill/issues/11061)) ([63cb46d](https://github.com/windmill-labs/windmill/commit/63cb46d7bb9db1d996aa38e06bd3afebc60111bd)) +* live queue status per tag and bounded queue metric charts ([#11067](https://github.com/windmill-labs/windmill/issues/11067)) ([569adb8](https://github.com/windmill-labs/windmill/commit/569adb85c1885d289e80a70a166f0f74e6d5ba83)) +* **otel:** read the OTLP metrics temporality preference ([#11064](https://github.com/windmill-labs/windmill/issues/11064)) ([2f88769](https://github.com/windmill-labs/windmill/commit/2f8876908719b3640d7cfc9364a7b0f7145fc356)) +* **otel:** support standard OTEL resource attribute env vars ([#10974](https://github.com/windmill-labs/windmill/issues/10974)) ([0a40eea](https://github.com/windmill-labs/windmill/commit/0a40eea37a7dbde5fc4760d6333d81186dfee255)) +* report script metadata with no content file in wmill lint ([#11053](https://github.com/windmill-labs/windmill/issues/11053)) ([8820b9f](https://github.com/windmill-labs/windmill/commit/8820b9fc644c6620c50517cd0d902015e2e670e2)) +* show the workspace an operator is in, and let them switch ([#11059](https://github.com/windmill-labs/windmill/issues/11059)) ([385086f](https://github.com/windmill-labs/windmill/commit/385086ffc21c72cd07624584932a4a301c23a732)) +* tuck other users' spaces into a collapsible home tree row ([#11073](https://github.com/windmill-labs/windmill/issues/11073)) ([d87f089](https://github.com/windmill-labs/windmill/commit/d87f089288996af9ea7e3b017a5ef35d4ded880d)) + + +### Bug Fixes + +* **ai-chat:** test_run_flow could test a different flow than the one asked ([#11066](https://github.com/windmill-labs/windmill/issues/11066)) ([fa73539](https://github.com/windmill-labs/windmill/commit/fa73539839071491fb2cbad9242f52ad22b975bf)) +* bound list_jobs runtime and paginate runs on the sorted column ([#11072](https://github.com/windmill-labs/windmill/issues/11072)) ([f517402](https://github.com/windmill-labs/windmill/commit/f51740253871960b55ab2aa8989e8df3fbde0351)) +* **frontend:** clear the flow graph selection through xyflow's store ([#11056](https://github.com/windmill-labs/windmill/issues/11056)) ([b4be8bc](https://github.com/windmill-labs/windmill/commit/b4be8bc5354fbd3a47c267c331aaf603c0f90e6e)) +* **frontend:** recompute dataflow edges when selecting a step ([#11070](https://github.com/windmill-labs/windmill/issues/11070)) ([08d876a](https://github.com/windmill-labs/windmill/commit/08d876aebf32ebb995a8c1839aa87794c0176bed)) +* **frontend:** restore heading sizes in note markdown and keep group notes on id change ([#11047](https://github.com/windmill-labs/windmill/issues/11047)) ([e63072c](https://github.com/windmill-labs/windmill/commit/e63072c216383700a23504be89782f4c69657174)) +* give every table a primary key so the db can be logically replicated ([#11036](https://github.com/windmill-labs/windmill/issues/11036)) ([e62bfdc](https://github.com/windmill-labs/windmill/commit/e62bfdcd8c6f1389601ccd2b5809c5eff0ff262e)) +* keep an app's deployed policy on wmill push ([#11049](https://github.com/windmill-labs/windmill/issues/11049)) ([0af7675](https://github.com/windmill-labs/windmill/commit/0af7675588300863883a99c7f14bc53cafec8a7e)) +* refuse cross-site GET requests that run Hub scripts ([#11054](https://github.com/windmill-labs/windmill/issues/11054)) ([ab9efc8](https://github.com/windmill-labs/windmill/commit/ab9efc897cc94d61a67263772806b15e9225cece)) +* skip the deploy PR when the git sync push committed nothing ([#11076](https://github.com/windmill-labs/windmill/issues/11076)) ([8ecbd33](https://github.com/windmill-labs/windmill/commit/8ecbd339eef7314a93d599fcea4377299d4c493d)) +* space the trailing AI settings cards ([#11044](https://github.com/windmill-labs/windmill/issues/11044)) ([5d7eed1](https://github.com/windmill-labs/windmill/commit/5d7eed1c02b0966289cc8cca00a15d76dad187a5)) +* stop uv pip compile emitting lockfile annotations ([#11042](https://github.com/windmill-labs/windmill/issues/11042)) ([8aa8b7e](https://github.com/windmill-labs/windmill/commit/8aa8b7ee6c23f859f169637c0bfd3f8d964509e9)) +* surface why a private or untrusted git host is unreachable ([#11068](https://github.com/windmill-labs/windmill/issues/11068)) ([c57b18e](https://github.com/windmill-labs/windmill/commit/c57b18e46fcdd319213fe0a537cb18de418ca688)) + + +### Performance Improvements + +* index the FK columns that cascade on workspace delete ([#11052](https://github.com/windmill-labs/windmill/issues/11052)) ([9a563f6](https://github.com/windmill-labs/windmill/commit/9a563f6d72da28fe09b785cd0683e9698df72bba)) +* only write queue metrics when a tag's backlog changes ([#11055](https://github.com/windmill-labs/windmill/issues/11055)) ([9d75929](https://github.com/windmill-labs/windmill/commit/9d75929247ea2ec39286971fcbebf95d886194f3)) + ## [1.808.0](https://github.com/windmill-labs/windmill/compare/v1.807.0...v1.808.0) (2026-09-09) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 9a5516f50a..08184adb1c 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -1724,7 +1724,7 @@ version = "0.71.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f58bf3d7db68cfbac37cfc485a8d711e87e064c3d0fe0435b92f7a407f9d6b3" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "cexpr", "clang-sys", "itertools 0.13.0", @@ -1744,7 +1744,7 @@ version = "0.72.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "cexpr", "clang-sys", "itertools 0.13.0", @@ -1787,9 +1787,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.13.1" +version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" dependencies = [ "serde_core", ] @@ -5077,7 +5077,7 @@ version = "25.12.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "rustc_version 0.4.1", ] @@ -6463,11 +6463,32 @@ checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" dependencies = [ "displaydoc", "litemap", + "serde", "tinystr", "writeable", "zerovec", ] +[[package]] +name = "icu_locale_fallback" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "251af8e57c9400e3eb58242fe5b8b1152b2a64fdf4cf632f923c38ccee6f2fa9" +dependencies = [ + "icu_locale_core", + "icu_locale_fallback_data", + "icu_provider", + "potential_utf", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locale_fallback_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "decf2a22ec8fa68f1a0c1129a3f8583f8f8bc24e8b9ccbe98ead99f62a4dc3a8" + [[package]] name = "icu_normalizer" version = "2.3.0" @@ -6517,6 +6538,8 @@ checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" dependencies = [ "displaydoc", "icu_locale_core", + "serde", + "stable_deref_trait", "writeable", "yoke", "zerofrom", @@ -6524,6 +6547,28 @@ dependencies = [ "zerovec", ] +[[package]] +name = "icu_segmenter" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82d07aafccd67af15d02512a6adf5896fbc5ed00f2e99b471d2efa14016db3db" +dependencies = [ + "icu_collections", + "icu_locale_fallback", + "icu_provider", + "icu_segmenter_data", + "potential_utf", + "smallvec", + "utf8_iter", + "zerovec", +] + +[[package]] +name = "icu_segmenter_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae293c039020f9ec10710af98d29ce6aa2051486638b49c9a6409f3b4a9e98ad" + [[package]] name = "ident_case" version = "1.0.1" @@ -6644,7 +6689,7 @@ version = "0.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed3bd0ecfbb87805f538bb7b32e5239ca0763890c623e349860ecba69469f2bb" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "cfg-if", "libc", ] @@ -7217,7 +7262,7 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e668df13f2e97f3eed52d9301f6b1c4c1ccfccc30eab9e6628e4a8c1fc3546" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "bytes", "lazy_static", "libgssapi-sys", @@ -7266,7 +7311,7 @@ version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d8f1ea3f21fd3405dcaf6c9b5c1630af9afc422d9073ea39c5f6d6c772e08ed" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "libc", "plain", "redox_syscall 0.9.4", @@ -7906,7 +7951,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f27695f286b461da077b8c2f72f47feaa04ce3c3f9c0976257410e90e21208a" dependencies = [ "base64 0.22.1", - "bitflags 2.13.1", + "bitflags 2.13.2", "btoi", "byteorder", "bytes", @@ -7965,7 +8010,7 @@ version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "cfg-if", "libc", ] @@ -7976,7 +8021,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "cfg-if", "cfg_aliases", "libc", @@ -7988,7 +8033,7 @@ version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "cfg-if", "cfg_aliases", "libc", @@ -8000,7 +8045,7 @@ version = "0.31.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "cfg-if", "cfg_aliases", "libc", @@ -8467,7 +8512,7 @@ version = "6.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cc3cbf698f9438986c11a880c90a6d04b9de27575afd28bbf45b154b6c709e2" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "libc", "once_cell", "onig_sys", @@ -8532,7 +8577,7 @@ version = "0.10.81" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "cfg-if", "foreign-types", "libc", @@ -9485,6 +9530,8 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" dependencies = [ + "serde_core", + "writeable", "zerovec", ] @@ -9544,7 +9591,7 @@ version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" dependencies = [ - "toml_edit 0.25.13+spec-1.1.0", + "toml_edit 0.25.14+spec-1.1.0", ] [[package]] @@ -9633,7 +9680,7 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc5b72d8145275d844d4b5f6d4e1eef00c8cd889edb6035c21675d1bb1f45c9f" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "chrono", "flate2", "hex", @@ -9647,7 +9694,7 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "239df02d8349b06fc07398a3a1697b06418223b1c7725085e801e7c0fc6a12ec" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "chrono", "hex", ] @@ -9734,7 +9781,7 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57206b407293d2bcd3af849ce869d52068623f19e1b5ff8e8778e3309439682b" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "getopts", "memchr", "unicase", @@ -10028,7 +10075,7 @@ version = "11.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", ] [[package]] @@ -10162,7 +10209,7 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", ] [[package]] @@ -10171,7 +10218,7 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "737970939a87c6fa31e7acad13307bccbb017a073b695b6089a2c484f929e20e" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", ] [[package]] @@ -10524,9 +10571,9 @@ dependencies = [ [[package]] name = "rmcp-macros" -version = "3.2.0" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdf1c49bd4d52014b94db0877410db273c2008f01628b0252a2e9460ad9b7fda" +checksum = "873b730df6f0a9b74b13eb514e0dca4c2db0d8b68b74af98a2e9bf3f9d436585" dependencies = [ "darling 0.24.1", "proc-macro2", @@ -10741,7 +10788,7 @@ version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "errno", "libc", "linux-raw-sys 0.4.15", @@ -10754,7 +10801,7 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "errno", "libc", "linux-raw-sys 0.12.1", @@ -11241,7 +11288,7 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "core-foundation 0.9.4", "core-foundation-sys", "libc", @@ -11254,7 +11301,7 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "core-foundation 0.10.1", "core-foundation-sys", "libc", @@ -12017,7 +12064,7 @@ dependencies = [ "atoi", "base64 0.22.1", "bigdecimal", - "bitflags 2.13.1", + "bitflags 2.13.2", "byteorder", "bytes", "chrono", @@ -12061,7 +12108,7 @@ dependencies = [ "atoi", "base64 0.22.1", "bigdecimal", - "bitflags 2.13.1", + "bitflags 2.13.2", "byteorder", "chrono", "crc", @@ -12335,7 +12382,7 @@ version = "15.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65c25af97d53cf8aab66a6c68f3418663313fc969ad267fc2a4d19402c329be1" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "is-macro", "num-bigint", "once_cell", @@ -12391,7 +12438,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "017d06ea85008234aa9fb34d805c7dc563f2ea6e03869ed5ac5a2dc27d561e4d" dependencies = [ "arrayvec", - "bitflags 2.13.1", + "bitflags 2.13.2", "either", "num-bigint", "phf 0.11.3", @@ -12724,7 +12771,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "01198a2debb237c62b6826ec7081082d951f46dbb64b0e8c7649a452230d1dfc" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "byteorder", "enum-as-inner", "libc", @@ -12752,7 +12799,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "core-foundation 0.9.4", "system-configuration-sys", ] @@ -13011,11 +13058,11 @@ dependencies = [ [[package]] name = "textwrap" -version = "0.16.2" +version = "0.16.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c13547615a44dc9c452a8a534638acdf07120d4b6847c8178705da06306a3057" +checksum = "b81c0cb5fce14f53e49c1d4da0c508334ff12040221bb8ab01b2dabd91d04b6e" dependencies = [ - "unicode-linebreak", + "icu_segmenter", "unicode-width 0.2.2", ] @@ -13197,6 +13244,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" dependencies = [ "displaydoc", + "serde_core", "zerovec", ] @@ -13559,9 +13607,9 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.25.13+spec-1.1.0" +version = "0.25.14+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6975367e4d2ef766d86af01ffad14b622fecc8d4357a998fbc4deb6e9bacaf9b" +checksum = "d2195eec204e2764644a4ea619704f9fbe5e0673038eded55ad9956f24fca0cc" dependencies = [ "indexmap 2.14.2", "toml_datetime 1.1.1+spec-1.1.0", @@ -13706,7 +13754,7 @@ checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ "async-compression", "base64 0.22.1", - "bitflags 2.13.1", + "bitflags 2.13.2", "bytes", "futures-core", "futures-util", @@ -14130,12 +14178,6 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" -[[package]] -name = "unicode-linebreak" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f" - [[package]] name = "unicode-normalization" version = "0.1.25" @@ -14335,9 +14377,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.26.0" +version = "1.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5772d71c9be8a8a6ac2117d949c5b224c1b72241bb611d9a3012edcf8af7812" +checksum = "2ef6dac1e96601b4fb3acccccff2139741fcb757cb9a36089bf5be91cfb285ce" dependencies = [ "getrandom 0.4.3", "js-sys", @@ -14352,7 +14394,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33995a1fee055ff743281cde33a41f0d618ee0bdbe8bdf6859e11864499c2595" dependencies = [ "bindgen 0.71.1", - "bitflags 2.13.1", + "bitflags 2.13.2", "fslock", "gzip-header", "home", @@ -14747,7 +14789,7 @@ dependencies = [ [[package]] name = "windmill" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "async-nats", @@ -14835,7 +14877,7 @@ dependencies = [ [[package]] name = "windmill-ai" -version = "1.808.0" +version = "1.809.0" dependencies = [ "async-stream", "async-trait", @@ -14868,7 +14910,7 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.808.0" +version = "1.809.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14881,7 +14923,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "argon2", @@ -15021,7 +15063,7 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.808.0" +version = "1.809.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15044,7 +15086,7 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.808.0" +version = "1.809.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15061,7 +15103,7 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15087,7 +15129,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.808.0" +version = "1.809.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -15097,7 +15139,7 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.808.0" +version = "1.809.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15114,7 +15156,7 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.808.0" +version = "1.809.0" dependencies = [ "axum 0.8.9", "base64 0.22.1", @@ -15136,7 +15178,7 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15159,7 +15201,7 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.808.0" +version = "1.809.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15175,7 +15217,7 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.808.0" +version = "1.809.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15197,7 +15239,7 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.808.0" +version = "1.809.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15218,7 +15260,7 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.808.0" +version = "1.809.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15232,7 +15274,7 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "async-nats", @@ -15267,7 +15309,7 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15292,7 +15334,7 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.808.0" +version = "1.809.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15320,7 +15362,7 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15342,7 +15384,7 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.808.0" +version = "1.809.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15362,7 +15404,7 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.808.0" +version = "1.809.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15400,7 +15442,7 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15428,7 +15470,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.808.0" +version = "1.809.0" dependencies = [ "lazy_static", "serde", @@ -15440,7 +15482,7 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.808.0" +version = "1.809.0" dependencies = [ "argon2", "axum 0.8.9", @@ -15464,7 +15506,7 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.808.0" +version = "1.809.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15478,7 +15520,7 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.808.0" +version = "1.809.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15513,7 +15555,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.808.0" +version = "1.809.0" dependencies = [ "chrono", "lazy_static", @@ -15527,7 +15569,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15546,7 +15588,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.808.0" +version = "1.809.0" dependencies = [ "aes-gcm", "aho-corasick", @@ -15563,7 +15605,7 @@ dependencies = [ "axum 0.8.9", "backon", "base64 0.22.1", - "bitflags 2.13.1", + "bitflags 2.13.2", "bytes", "chrono", "chrono-tz", @@ -15652,7 +15694,7 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.808.0" +version = "1.809.0" dependencies = [ "chrono", "futures", @@ -15672,7 +15714,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.808.0" +version = "1.809.0" dependencies = [ "regex", "serde", @@ -15687,7 +15729,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "astral-tokio-tar", @@ -15714,7 +15756,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "futures", @@ -15731,7 +15773,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.808.0" +version = "1.809.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -15747,7 +15789,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "async-trait", @@ -15768,7 +15810,7 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "async-trait", @@ -15799,7 +15841,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "arc-swap", @@ -15824,7 +15866,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "async-stream", @@ -15859,7 +15901,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "futures", @@ -15877,7 +15919,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.808.0" +version = "1.809.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -15886,7 +15928,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "lazy_static", @@ -15898,7 +15940,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "serde_json", @@ -15910,7 +15952,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "gosyn", @@ -15922,7 +15964,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "lazy_static", @@ -15934,7 +15976,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "serde_json", @@ -15946,7 +15988,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "nu-parser", @@ -15957,7 +15999,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15968,7 +16010,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15980,7 +16022,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "rustpython-ast", @@ -15991,7 +16033,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "async-recursion", @@ -16013,7 +16055,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "serde_json", @@ -16025,7 +16067,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "lazy_static", @@ -16039,7 +16081,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -16056,7 +16098,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "lazy_static", @@ -16069,7 +16111,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "serde", @@ -16081,7 +16123,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "lazy_static", @@ -16099,7 +16141,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -16115,7 +16157,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "rustpython-ast", @@ -16131,7 +16173,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "lazy_static", @@ -16145,7 +16187,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "async-recursion", @@ -16184,7 +16226,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "const_format", @@ -16224,7 +16266,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.808.0" +version = "1.809.0" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -16235,7 +16277,7 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "async-recursion", @@ -16270,7 +16312,7 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "async-trait", @@ -16294,7 +16336,7 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "async-trait", @@ -16327,7 +16369,7 @@ dependencies = [ [[package]] name = "windmill-trigger-amqp" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "async-trait", @@ -16354,7 +16396,7 @@ dependencies = [ [[package]] name = "windmill-trigger-azure" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "async-trait", @@ -16387,7 +16429,7 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "async-trait", @@ -16407,7 +16449,7 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "async-trait", @@ -16441,7 +16483,7 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "async-trait", @@ -16477,7 +16519,7 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "async-trait", @@ -16500,7 +16542,7 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "async-trait", @@ -16524,7 +16566,7 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "async-nats", @@ -16548,7 +16590,7 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "async-trait", @@ -16583,7 +16625,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "async-trait", @@ -16611,7 +16653,7 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "async-trait", @@ -16636,10 +16678,10 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", - "bitflags 2.13.1", + "bitflags 2.13.2", "chrono", "hex", "itertools 0.14.0", @@ -16655,7 +16697,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "async-once-cell", @@ -16773,7 +16815,7 @@ dependencies = [ [[package]] name = "windmill-worker-volumes" -version = "1.808.0" +version = "1.809.0" dependencies = [ "bytes", "futures", @@ -17024,7 +17066,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d24d6bcc7f734a4091ecf8d7a64c5f7d7066f45585c1861eba06449909609c8a" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "widestring", "windows-sys 0.52.0", ] @@ -17541,6 +17583,7 @@ dependencies = [ "displaydoc", "yoke", "zerofrom", + "zerovec", ] [[package]] @@ -17549,6 +17592,7 @@ version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" dependencies = [ + "serde", "yoke", "zerofrom", "zerovec-derive", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index e1f5aafac2..a6b42d3e7a 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.808.0" +version = "1.809.0" authors.workspace = true edition.workspace = true @@ -88,7 +88,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"] [workspace.package] -version = "1.808.0" +version = "1.809.0" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/parsers/windmill-parser-wasm/Cargo.lock b/backend/parsers/windmill-parser-wasm/Cargo.lock index 98631d75cd..5c6ad27103 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.lock +++ b/backend/parsers/windmill-parser-wasm/Cargo.lock @@ -6191,7 +6191,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill-common" -version = "1.808.0" +version = "1.809.0" dependencies = [ "aho-corasick", "anyhow", @@ -6274,7 +6274,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.808.0" +version = "1.809.0" dependencies = [ "proc-macro2", "quote", @@ -6286,7 +6286,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.808.0" +version = "1.809.0" dependencies = [ "convert_case", "serde", @@ -6295,7 +6295,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "lazy_static", @@ -6307,7 +6307,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "serde_json", @@ -6319,7 +6319,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "gosyn", @@ -6331,7 +6331,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "lazy_static", @@ -6343,7 +6343,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "serde_json", @@ -6355,7 +6355,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "nu-parser", @@ -6366,7 +6366,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6377,7 +6377,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6389,7 +6389,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6400,7 +6400,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "async-recursion", @@ -6422,7 +6422,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "serde_json", @@ -6434,7 +6434,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "lazy_static", @@ -6448,7 +6448,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "convert_case", @@ -6465,7 +6465,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "lazy_static", @@ -6478,7 +6478,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "serde", @@ -6490,7 +6490,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "lazy_static", @@ -6508,7 +6508,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -6524,7 +6524,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6540,7 +6540,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "getrandom 0.2.17", @@ -6572,7 +6572,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "lazy_static", @@ -6586,7 +6586,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.808.0" +version = "1.809.0" dependencies = [ "anyhow", "bitflags", diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index 1dee3f3398..f91b32e726 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -12,7 +12,7 @@ resolver = "2" members = ["."] [workspace.package] -version = "1.808.0" +version = "1.809.0" edition = "2021" authors = ["Ruben Fiszel "] diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 09d0838def..a43135d440 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.808.0 + version: 1.809.0 title: Windmill API contact: diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 990c48f014..d5efec8ff0 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.808.0"; +export const VERSION = "v1.809.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/src/core/constants.ts b/cli/src/core/constants.ts index 54fd3a6a8d..228d0444a5 100644 --- a/cli/src/core/constants.ts +++ b/cli/src/core/constants.ts @@ -10,4 +10,4 @@ export const WM_FORK_PREFIX = "wm-fork"; // (e.g. utils.ts) can read it without importing main.ts and creating a circular // dependency (main → workspace → utils → main) that triggers a TDZ. // Re-exported from main.ts for backwards compatibility. -export const VERSION = "1.808.0"; +export const VERSION = "1.809.0"; diff --git a/frontend/package-lock.json b/frontend/package-lock.json index d5f11bb9e0..4c9b9350ef 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "@windmill-labs/components", - "version": "1.808.0", + "version": "1.809.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@windmill-labs/components", - "version": "1.808.0", + "version": "1.809.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { diff --git a/frontend/package.json b/frontend/package.json index e05851cc58..9b7df8228a 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "@windmill-labs/components", - "version": "1.808.0", + "version": "1.809.0", "scripts": { "dev": "vite dev", "dev:ui-builder": "mv static/ui_builder static/ui_builder.dev-disabled 2>/dev/null || true ; trap 'mv static/ui_builder.dev-disabled static/ui_builder 2>/dev/null || true' EXIT ; vite dev", diff --git a/lsp/Pipfile b/lsp/Pipfile index 3ec5759b49..8b92d34a5b 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,7 +4,7 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.808.0" +wmill = ">=1.809.0" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index 1d7af0ebb0..11c68626e1 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: '3.0.3' info: - version: 1.808.0 + version: 1.809.0 title: OpenFlow Spec contact: name: Ruben Fiszel diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index 7aa128b0d5..52fdfae90d 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. - ModuleVersion = '1.808.0' + ModuleVersion = '1.809.0' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index 3c89ddaff2..ed68a6c23d 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.808.0" +version = "1.809.0" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index da976fca03..bba03a4eba 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.808.0", + "version": "1.809.0", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./sqlUtils.ts", "./client.ts", "./wacError.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index 0cedb321cc..894921cc2c 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.808.0", + "version": "1.809.0", "author": "Ruben Fiszel", "license": "Apache 2.0", "homepage": "https://github.com/windmill-labs/windmill/tree/main/typescript-client#readme", diff --git a/version.txt b/version.txt index ec20da5139..667afa2fde 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.808.0 +1.809.0 diff --git a/windmill-yaml-validator/package-lock.json b/windmill-yaml-validator/package-lock.json index 67dee7ff01..35c2126a73 100644 --- a/windmill-yaml-validator/package-lock.json +++ b/windmill-yaml-validator/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-yaml-validator", - "version": "1.808.0", + "version": "1.809.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-yaml-validator", - "version": "1.808.0", + "version": "1.809.0", "license": "Apache 2.0", "dependencies": { "@stoplight/yaml": "^4.3.0", diff --git a/windmill-yaml-validator/package.json b/windmill-yaml-validator/package.json index c118a08b8e..c55783af6b 100644 --- a/windmill-yaml-validator/package.json +++ b/windmill-yaml-validator/package.json @@ -1,6 +1,6 @@ { "name": "windmill-yaml-validator", - "version": "1.808.0", + "version": "1.809.0", "description": "YAML validator for Windmill flow, schedule, and trigger files", "main": "dist/index.js", "types": "dist/index.d.ts", From f915ed6a46e14341ddb213e869090edb68763032 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 11 Sep 2026 08:01:26 +0200 Subject: [PATCH 072/144] fix: attach TLS to gRPC OTLP exporters for https endpoints (#11078) * fix: attach TLS to gRPC OTLP exporters for https endpoints Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01RbLzVfFDZ9pjBGHSzCSrNZ * chore: bump ee-repo-ref for the gRPC TLS resolution test Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01RbLzVfFDZ9pjBGHSzCSrNZ * chore: update ee-repo-ref to afd59490e2ca3c375d64cf4a91041763c7de4766 This commit updates the EE repository reference after PR #790 was merged in windmill-ee-private. Previous ee-repo-ref: b6dd68beb144ecf1b978172396ff8c8bb27ae0c2 New ee-repo-ref: afd59490e2ca3c375d64cf4a91041763c7de4766 Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: windmill-internal-app[bot] --- backend/ee-repo-ref.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index d426e06c2c..bf25b58207 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -e92abc9d1fba3ba898640df0cfeb0af8a50849b4 +afd59490e2ca3c375d64cf4a91041763c7de4766 From b156778da24e3827f723e503f80df68de87ddf7e Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 11 Sep 2026 08:01:54 +0200 Subject: [PATCH 073/144] fix: support gzip and zstd compression for OTLP export over gRPC (#11077) Claude-Session: https://claude.ai/code/session_01RbLzVfFDZ9pjBGHSzCSrNZ Co-authored-by: Claude Opus 5 (1M context) --- backend/Cargo.lock | 2 ++ backend/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 08184adb1c..26ee0710f3 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -13670,6 +13670,7 @@ dependencies = [ "axum 0.8.9", "base64 0.22.1", "bytes", + "flate2", "h2 0.4.19", "http 1.5.0", "http-body 1.1.0", @@ -13689,6 +13690,7 @@ dependencies = [ "tower-layer", "tower-service", "tracing", + "zstd", ] [[package]] diff --git a/backend/Cargo.toml b/backend/Cargo.toml index a6b42d3e7a..be7ab2d192 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -701,7 +701,7 @@ async-stream = "^0" opentelemetry = "0.30.0" tracing-opentelemetry = "0.31.0" opentelemetry_sdk = { version = "0.30.0", features = ["rt-tokio", "testing"] } -opentelemetry-otlp = { version = "0.30.0", features = ["grpc-tonic", "tls", "http-proto"] } +opentelemetry-otlp = { version = "0.30.0", features = ["grpc-tonic", "tls", "http-proto", "gzip-tonic", "zstd-tonic"] } opentelemetry-appender-tracing = "0.30.0" opentelemetry-semantic-conventions = { version = "0.30.0", features = ["semconv_experimental"] } opentelemetry-proto = { version = "0.30.0", features = ["with-serde", "gen-tonic"] } From e6d4f44a6122dab47fbee2a2a2b4330a62841bed Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 11 Sep 2026 09:25:24 +0200 Subject: [PATCH 074/144] fix: show symlinked files in the git repo viewer (#11081) * docs: describe symlink handling in the repo viewer's hub script Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01FPgYLu8Eiq58Bv2ev2YRYN * docs: describe the symlink budget in the repo viewer's hub script Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01FPgYLu8Eiq58Bv2ev2YRYN * docs: link the repo viewer script's hub page and soften the skip-log claim Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01FPgYLu8Eiq58Bv2ev2YRYN * docs: charge the symlink budget before a link is resolved Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01FPgYLu8Eiq58Bv2ev2YRYN * fix: point the git repo viewer at the symlink-following clone script Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01FPgYLu8Eiq58Bv2ev2YRYN * docs: note that a new pin doesn't refresh commits already uploaded Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01FPgYLu8Eiq58Bv2ev2YRYN --------- Co-authored-by: Claude Opus 5 (1M context) --- ...repo_and_upload_to_instance_storage.bun.ts | 390 ------------------ docs/git-repo-viewer-hub-script.md | 113 ++++- frontend/src/lib/hubPaths.json | 2 +- 3 files changed, 103 insertions(+), 402 deletions(-) delete mode 100644 docs/clone_repo_and_upload_to_instance_storage.bun.ts diff --git a/docs/clone_repo_and_upload_to_instance_storage.bun.ts b/docs/clone_repo_and_upload_to_instance_storage.bun.ts deleted file mode 100644 index 485ac0c766..0000000000 --- a/docs/clone_repo_and_upload_to_instance_storage.bun.ts +++ /dev/null @@ -1,390 +0,0 @@ -import * as wmillclient from "windmill-client"; -import { basename, join } from "node:path"; -import { existsSync, rmSync } from "fs"; -import process from "process"; -import { spawn } from 'child_process'; -import * as fs_async from 'fs/promises'; -import * as fs from 'node:fs'; - -const UPLOAD_CONCURRENCY = 16; -const CLONE_MARKER_FILE = ".windmill_clone_complete"; - -type GitRepository = { - url: string; - branch: string; - folder: string; - gpg_key: any; - is_github_app: boolean; -}; - -export async function main( - resource_path: string, - workspace: string, - git_ssh_identity?: string[], - commit?: string -) { - let clonedRepoPath: string | undefined; - - try { - console.log("Starting git clone and Blob storage upload process"); - - // Get the git repository resource - const repo_resource: GitRepository = await wmillclient.getResource(resource_path); - - const cwd = process.cwd(); - - if (git_ssh_identity) { - process.env.GIT_SSH_COMMAND = await get_git_ssh_cmd(cwd, git_ssh_identity) - } - - // Handle GitHub App authentication if needed - if (repo_resource.is_github_app) { - const token = await get_gh_app_token(); - repo_resource.url = prependTokenToGitHubUrl(repo_resource.url, token); - } - - process.env["HOME"] = "."; - process.env.GIT_TERMINAL_PROMPT = "0"; - - // Clone the repository - const { repo_name, commitHash } = await git_clone(cwd, repo_resource, commit); - clonedRepoPath = join(cwd, repo_name); - - // Remove .git directory to avoid uploading git history - const gitDir = join(clonedRepoPath, ".git"); - if (existsSync(gitDir)) { - rmSync(gitDir, { recursive: true, force: true }); - console.log("Removed .git directory"); - } - - // Upload to S3 - const s3Path = `gitrepos/${workspace}/${resource_path}/${commitHash}`; - const fileCount = await uploadDirectoryToS3(clonedRepoPath, s3Path, workspace); - - return { - success: true, - message: "Repository cloned and uploaded to S3 successfully", - s3_path: s3Path, - commit_hash: commitHash, - file_count: fileCount, - }; - - } catch (error) { - console.error("Error in git clone and upload:", error); - throw error; - } finally { - // Clean up cloned repository - if (clonedRepoPath && existsSync(clonedRepoPath)) { - rmSync(clonedRepoPath, { recursive: true, force: true }); - console.log("Cleaned up cloned repository"); - } - } -} - -async function get_git_ssh_cmd(cwd: string, git_ssh_identity: string[]): Promise { - const sshIdFiles = await Promise.all( - git_ssh_identity.map(async (varPath, i) => { - const filePath = join(cwd, `./ssh_id_priv_${i}`); - - try { - // Get variable value using windmill - let content = await wmillclient.getVariable(varPath); - content += '\n'; - - // Write file with content - await fs_async.writeFile(filePath, content, { encoding: 'utf8' }); - - // Set file permissions to 0o600 (read/write for owner only) - await fs_async.chmod(filePath, 0o600); - - // Escape single quotes for shell command - const escapedPath = filePath.replace(/'/g, "'\\''"); - return ` -i '${escapedPath}'`; - } catch (error) { - console.error( - `Variable ${varPath} not found for git ssh identity: ${error}` - ); - return ''; - } - }) - ); - - const gitSshCmd = `ssh -o StrictHostKeyChecking=no${sshIdFiles.join('')}`; - return gitSshCmd; -} - -async function git_clone( - cwd: string, - repo_resource: GitRepository, - commit?: string, -): Promise<{ repo_name: string; commitHash: string }> { - if (commit) { - return git_clone_at_commit(cwd, repo_resource, commit); - } else { - return git_clone_at_latest(cwd, repo_resource); - } -} - -async function git_clone_at_commit( - cwd: string, - repo_resource: GitRepository, - commit: string, -): Promise<{ repo_name: string; commitHash: string }> { - let repo_url = repo_resource.url; - const subfolder = repo_resource.folder ?? ""; - let branch = repo_resource.branch ?? ""; - const repo_name = basename(repo_url, ".git"); - - const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?.+)\)/); - if (azureMatch) { - console.log("Fetching Azure DevOps access token..."); - const azureResource = await wmillclient.getResource(azureMatch.groups.url); - const response = await fetch( - `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`, - { - method: "POST", - body: new URLSearchParams({ - client_id: azureResource.azureClientId, - client_secret: azureResource.azureClientSecret, - grant_type: "client_credentials", - resource: "499b84ac-1321-427f-aa17-267ca6975798/.default", - }), - } - ); - const { access_token } = await response.json(); - repo_url = repo_url.replace(azureMatch[0], access_token); - } - - const repoPath = join(cwd, repo_name); - await fs_async.mkdir(repoPath, { recursive: true }); - - process.chdir(repoPath); - - let args = ['init', '--quiet'] - if (branch) { - args.push(`--initial-branch=${branch}`) - } - await runCommand(undefined, 'git', ...args); - - await runCommand(0, 'git', 'remote', 'add', 'origin', repo_url); - - await runCommand(undefined, 'git', 'fetch', '--depth=1', '--quiet', 'origin', commit); - - await runCommand(undefined, 'git', 'checkout', '--quiet', 'FETCH_HEAD'); - - const commitHash = (await runCommand(undefined, "git", "rev-parse", "HEAD")).trim(); - - // Return to original directory - process.chdir(cwd); - - return { repo_name, commitHash }; -} - -async function git_clone_at_latest( - cwd: string, - repo_resource: GitRepository -): Promise<{ repo_name: string; commitHash: string }> { - let repo_url = repo_resource.url; - const subfolder = repo_resource.folder ?? ""; - let branch = repo_resource.branch ?? ""; - const repo_name = basename(repo_url, ".git"); - - // Handle Azure DevOps token if needed - const azureMatch = repo_url.match(/AZURE_DEVOPS_TOKEN\((?.+)\)/); - if (azureMatch) { - console.log("Fetching Azure DevOps access token..."); - const azureResource = await wmillclient.getResource(azureMatch.groups.url); - const response = await fetch( - `https://login.microsoftonline.com/${azureResource.azureTenantId}/oauth2/token`, - { - method: "POST", - body: new URLSearchParams({ - client_id: azureResource.azureClientId, - client_secret: azureResource.azureClientSecret, - grant_type: "client_credentials", - resource: "499b84ac-1321-427f-aa17-267ca6975798/.default", - }), - } - ); - const { access_token } = await response.json(); - repo_url = repo_url.replace(azureMatch[0], access_token); - } - - const args = ["clone", "--quiet", "--depth", "1"]; - if (subfolder !== "") args.push("--sparse"); - if (branch !== "") args.push("--branch", branch); - args.push(repo_url, repo_name); - - await runCommand(-1, "git", ...args); - - const fullPath = join(cwd, repo_name); - process.chdir(fullPath); - - if (subfolder !== "") { - await runCommand(undefined, "git", "sparse-checkout", "add", subfolder); - const subfolderPath = join(fullPath, subfolder); - - if (!existsSync(subfolderPath)) { - throw new Error(`Subfolder ${subfolder} does not exist.`); - } - - process.chdir(subfolderPath); - } - - // Get the commit hash - const commitHash = (await runCommand(undefined, "git", "rev-parse", "HEAD")).trim(); - - // Return to original directory - process.chdir(cwd); - - return { repo_name, commitHash }; -} - -async function uploadDirectoryToS3( - directoryPath: string, - s3BasePath: string, - workspace: string, -): Promise { - console.log(`Uploading ${directoryPath} -> ${s3BasePath}`); - - // Walk once into a flat task list so we can drive a bounded-concurrency pool. - const tasks: { localPath: string; s3Key: string }[] = []; - function walk(dir: string, s3Path: string) { - for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { - const fullPath = join(dir, entry.name); - const s3Key = s3Path ? `${s3Path}/${entry.name}` : entry.name; - if (entry.isDirectory()) { - walk(fullPath, s3Key); - } else if (entry.isFile()) { - tasks.push({ localPath: fullPath, s3Key }); - } - } - } - walk(directoryPath, s3BasePath); - console.log(`Discovered ${tasks.length} files to upload`); - - let nextIndex = 0; - let uploaded = 0; - let lastReport = 0; - async function worker() { - while (true) { - const idx = nextIndex++; - if (idx >= tasks.length) return; - const { localPath, s3Key } = tasks[idx]; - const fileContent = fs.readFileSync(localPath); - const blob = new Blob([fileContent], { type: 'application/octet-stream' }); - await wmillclient.HelpersService.gitRepoViewerFileUpload({ - workspace, - fileKey: s3Key, - requestBody: blob, - }); - uploaded++; - if (uploaded - lastReport >= 25 || uploaded === tasks.length) { - lastReport = uploaded; - console.log(`Uploaded ${uploaded} / ${tasks.length} files`); - } - } - } - await Promise.all( - Array.from({ length: Math.min(UPLOAD_CONCURRENCY, tasks.length) }, () => worker()) - ); - - // Marker is the LAST write — its presence is what the viewer checks for. - const markerKey = `${s3BasePath}/${CLONE_MARKER_FILE}`; - const markerBody = JSON.stringify({ - completed_at: new Date().toISOString(), - file_count: tasks.length, - }); - await wmillclient.HelpersService.gitRepoViewerFileUpload({ - workspace, - fileKey: markerKey, - requestBody: new Blob([markerBody], { type: 'application/json' }), - }); - console.log(`Wrote completion marker: ${markerKey}`); - - return tasks.length; -} - -function runCommand(secret_position: number | undefined, cmd: string, ...args: string[]): Promise { - const nargs = secret_position != undefined ? args.slice() : args; - if (secret_position && secret_position < 0) - secret_position = nargs.length - 1 + secret_position; - - let secret: string | undefined = undefined; - if (secret_position != undefined) { - nargs[secret_position] = "***"; - secret = args[secret_position]; - } - console.log(`Running shell command: '${cmd} ${nargs.join(" ")} ...'`); - - return new Promise((resolve, reject) => { - const process = spawn(cmd, args); - - let stdout = ''; - let stderr = ''; - - process.stdout.on('data', (data) => { - stdout += data.toString(); - }); - - process.stderr.on('data', (data) => { - stderr += data.toString(); - }); - - process.on('error', (error) => { - let errorString = error.toString(); - if (secret) errorString = errorString.replace(secret, "***"); - console.log(`Shell command FAILED: ${cmd}`, errorString); - const e = new Error( - `SH command '${cmd} ${nargs.join(" ")}' failed: ${errorString}` - ); - reject(e); - }); - - process.on('close', (code) => { - if (stdout.length > 0) { - console.log("Shell stdout:", stdout); - } - if (stderr.length > 0) { - console.log("Shell stderr:", stderr); - } - if (code === 0) { - console.log(`Shell command completed successfully: ${cmd}`); - resolve(stdout); - } else { - reject(new Error(`Command failed with code ${code}: ${stderr}`)); - } - }); - }); -} - -async function get_gh_app_token() { - const workspace = process.env["WM_WORKSPACE"]; - const jobToken = process.env["WM_TOKEN"]; - const baseUrl = - process.env["BASE_INTERNAL_URL"] ?? - process.env["BASE_URL"] ?? - "http://localhost:8000"; - const url = `${baseUrl}/api/w/${workspace}/github_app/token`; - - const response = await fetch(url, { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${jobToken}`, - }, - body: JSON.stringify({ job_token: jobToken }), - }); - - if (!response.ok) { - const errorBody = await response.text().catch(() => ""); - throw new Error(`GitHub App token error (${response.status}): ${errorBody || response.statusText}`); - } - const data = await response.json(); - return data.token; -} - -function prependTokenToGitHubUrl(gitHubUrl: string, installationToken: string) { - const url = new URL(gitHubUrl); - return `https://x-access-token:${installationToken}@${url.hostname}${url.pathname}`; -} diff --git a/docs/git-repo-viewer-hub-script.md b/docs/git-repo-viewer-hub-script.md index 7bfbe8160f..a1b5ea1482 100644 --- a/docs/git-repo-viewer-hub-script.md +++ b/docs/git-repo-viewer-hub-script.md @@ -1,7 +1,10 @@ # Git repo viewer — hub script -The hub script `clone_repo_and_upload_to_instance_storage` is published from -`windmill-integrations` and pinned in `frontend/src/lib/hubPaths.json` as +The hub script `clone_repo_and_upload_to_instance_storage` +([hub page](https://hub.windmill.dev/scripts/windmill/13968)) is published from +`windmill-integrations` +(`hub/windmill/scripts/action/13968_clone_repo_and_upload_to_instance_storage/script.ts`) +and pinned in `frontend/src/lib/hubPaths.json` as `cloneRepoToS3forGitRepoViewer`. Hub paths are exact version pins, so editing the script means publishing a new version and repointing that entry. @@ -23,6 +26,11 @@ The repo viewer in the Windmill app expects the hub script to: 3. **Write a completion marker** as the very last action of a successful run, so the API and frontend can distinguish a fully-populated S3 directory from a partial / interrupted upload. +4. **Follow symlinks that stay inside the checkout.** Both the git clone and + the archive extraction keep a repository's symlinks as links, and + `Dirent.isFile()` / `isDirectory()` are both false for a link, so a walk + that only checks those drops every linked file and directory from the + viewer. See [Symlinks](#symlinks). The marker file the frontend looks for is `.windmill_clone_complete` at the root of the per-commit directory: @@ -43,28 +51,74 @@ after the walk completes: ```ts const UPLOAD_CONCURRENCY = 16 const CLONE_MARKER_FILE = ".windmill_clone_complete" +const MAX_SYMLINKED_ENTRIES = 20_000 +const MAX_SYMLINKED_BYTES = 512 * 1024 * 1024 async function uploadDirectoryToS3( directoryPath: string, s3BasePath: string, workspace: string, -) { +): Promise { console.log(`Uploading ${directoryPath} -> ${s3BasePath}`) // Walk the directory once, producing a flat list of (localPath, s3Key) pairs. const tasks: { localPath: string; s3Key: string }[] = [] - function walk(dir: string, s3Path: string) { + const root = fs.realpathSync(directoryPath) + // Real paths of the directories being descended through. + const ancestors = new Set() + // What entries reached through a link have cost so far; see Symlinks below. + let symlinkedEntries = 0 + let symlinkedBytes = 0 + let symlinkBudgetSpent = false + function chargeSymlinkBudget(relPath: string, entries: number, bytes: number): boolean { + if (symlinkBudgetSpent) return false + symlinkedEntries += entries + symlinkedBytes += bytes + if (symlinkedEntries <= MAX_SYMLINKED_ENTRIES && symlinkedBytes <= MAX_SYMLINKED_BYTES) { + return true + } + symlinkBudgetSpent = true + console.log( + `Skipping ${relPath} and every symlinked entry after it: symlinks reach more than ` + + `${MAX_SYMLINKED_ENTRIES} entries or ${MAX_SYMLINKED_BYTES / 2 ** 20} MiB` + ) + return false + } + function walk(dir: string, relDir: string, viaLink: boolean) { + ancestors.add(dir) for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { - const fullPath = join(dir, entry.name) - const s3Key = s3Path ? `${s3Path}/${entry.name}` : entry.name - if (entry.isDirectory()) { - walk(fullPath, s3Key) - } else if (entry.isFile()) { - tasks.push({ localPath: fullPath, s3Key }) + const relPath = relDir ? `${relDir}/${entry.name}` : entry.name + const linked = viaLink || entry.isSymbolicLink() + if (linked && !chargeSymlinkBudget(relPath, 1, 0)) continue + let localPath = join(dir, entry.name) + if (entry.isSymbolicLink()) { + const link = fs.readlinkSync(localPath) + try { + localPath = fs.realpathSync(localPath) + } catch (e: any) { + console.log(`Skipping symlink ${relPath} -> ${link}: cannot resolve target (${e.code})`) + continue + } + if (localPath !== root && !localPath.startsWith(root + sep)) { + console.log(`Skipping symlink ${relPath} -> ${link}: target is outside the repository`) + continue + } + } + const stat = fs.statSync(localPath) + if (stat.isDirectory() && ancestors.has(localPath)) { + console.log(`Skipping ${relPath}: links back to a directory it is inside`) + continue + } + if (linked && stat.isFile() && !chargeSymlinkBudget(relPath, 0, stat.size)) continue + if (stat.isDirectory()) { + walk(localPath, relPath, linked) + } else if (stat.isFile()) { + tasks.push({ localPath, s3Key: `${s3BasePath}/${relPath}` }) } } + ancestors.delete(dir) } - walk(directoryPath, s3BasePath) + walk(root, "", false) console.log(`Discovered ${tasks.length} files to upload`) @@ -114,9 +168,40 @@ async function uploadDirectoryToS3( requestBody: new Blob([markerBody], { type: "application/json" }), }) console.log(`Wrote completion marker: ${markerKey}`) + + return tasks.length } ``` +## Symlinks + +A link is resolved with `realpathSync` and followed only when its target lies +inside the checkout's real path. A file target is uploaded under the link's own +path; a directory target is walked as if it sat there, so +`inventories/prod/group_vars -> ../../shared/group_vars` shows up in the viewer +with its files. Everything else is skipped and logged: + +- **A target outside the checkout.** The repository chooses the target, and the + checkout sits in the job's working directory next to the ssh key + `get_git_ssh_cmd` writes (`../ssh_id_priv_0`) and the job's `args.json`. A + link to one of those, or to `/proc/self/environ` with the caller's + `WM_TOKEN`, would put it in storage for every reader of the resource. This + is why the walk does not follow links the way `aws s3 sync` does. +- **A target that cannot be resolved**: a dangling link, or a link loop + (`ELOOP`). +- **A directory that is already being walked higher up** (`loop -> .`, + `up -> ..`). The guard holds the real paths of the current descent only, as + `find -L` does, not every directory seen so far: a directory reachable + through two links is uploaded under both paths, as the checkout presents it. +- **Anything reached through a link once the budget is spent.** Because a + directory can be reached along many paths, two links to the next directory + at each level double the tree, and a repository a few dozen links deep would + expand past what the job can hold in memory. Every entry reached through a + link counts against a budget of 20,000 entries and 512 MiB. It is charged + before the link is resolved, so links that end up skipped count too, and + neither their work nor their log lines can multiply. Past the budget, the rest + are skipped with one log line. The checkout's own files are always uploaded. + ## Notes for review - **Concurrency level**: 16 is a starting point; tune based on instance @@ -127,6 +212,12 @@ async function uploadDirectoryToS3( paths on retry, so a partial upload + retry naturally heals. Old commit directories from before this patch are unreachable through the UI but still consume storage; an instance admin can prune them manually if desired. +- **A new pin doesn't refresh commits already uploaded**: the viewer keys + storage on the commit hash (`gitrepos/{workspace}/{resource_path}/{commit_hash}/`) + and only checks that the marker exists. So a commit uploaded by an earlier + script version keeps that version's tree (hub/28905's had no symlinks) until + the repository's head moves to a new commit, or an admin deletes that + commit's directory. - **Error propagation**: keep the existing `try/catch` in `main` so an upload failure surfaces in the job result and is shown in the new viewer error banner. diff --git a/frontend/src/lib/hubPaths.json b/frontend/src/lib/hubPaths.json index 77c1f829ce..7439ad7ada 100644 --- a/frontend/src/lib/hubPaths.json +++ b/frontend/src/lib/hubPaths.json @@ -12,5 +12,5 @@ "discordReport": "hub/9085/discord", "smtpReport": "hub/28242/smtp", "appReport": "hub/28243/app-report", - "cloneRepoToS3forGitRepoViewer": "hub/28905/clone_repo_and_upload_to_instance_storage" + "cloneRepoToS3forGitRepoViewer": "hub/28968/clone_repo_and_upload_to_instance_storage" } From fa53099e2b87a8676c5d6a18e77844e17ff45efd Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 11 Sep 2026 10:30:38 +0200 Subject: [PATCH 075/144] fix: let admins and background sync reach private git hosts (#11084) * fix: let admins and background sync reach private git hosts Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_011Hb4vHnVrCMqe8ZtNtzFSs * chore: point ee-repo-ref at the private git host change Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_011Hb4vHnVrCMqe8ZtNtzFSs * fix: treat any admin token as admin and pin git probe transports Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_011Hb4vHnVrCMqe8ZtNtzFSs * fix: pin git probe transports with a test and say what the caller check skips Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_011Hb4vHnVrCMqe8ZtNtzFSs * chore: update ee-repo-ref to eccad9f68bd7246cc81acb82bdb6c08fc6013f45 This commit updates the EE repository reference after PR #791 was merged in windmill-ee-private. Previous ee-repo-ref: 45ed1331a82dc15e6bdf15fd63517227f9160e21 New ee-repo-ref: eccad9f68bd7246cc81acb82bdb6c08fc6013f45 Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: windmill-internal-app[bot] --- backend/ee-repo-ref.txt | 2 +- backend/tests/git_sync_fork_credential.rs | 18 +- backend/windmill-common/src/ssrf.rs | 82 +++++---- backend/windmill-store/src/resources.rs | 200 +++++++++++++--------- 4 files changed, 183 insertions(+), 119 deletions(-) diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index bf25b58207..f3b1f4732b 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -afd59490e2ca3c375d64cf4a91041763c7de4766 +eccad9f68bd7246cc81acb82bdb6c08fc6013f45 diff --git a/backend/tests/git_sync_fork_credential.rs b/backend/tests/git_sync_fork_credential.rs index c594b31af6..10cf1e3735 100644 --- a/backend/tests/git_sync_fork_credential.rs +++ b/backend/tests/git_sync_fork_credential.rs @@ -251,23 +251,25 @@ async fn a_credential_is_not_served_over_a_downgraded_transport( Ok(()) } -/// A GitLab the server refuses to reach is the error reported, not the GitHub App -/// lookup that runs after it: for a self-managed GitLab on a private network, -/// "no GitHub App installation" names neither the host nor the setting that -/// allows it. +/// A GitLab the server cannot reach is the error reported, not the GitHub App +/// lookup that runs after it: for a self-managed GitLab behind a firewall or an +/// untrusted certificate, "no GitHub App installation" names neither the host +/// nor the cause. #[sqlx::test(fixtures("git_sync_fork_credential"))] -async fn a_refused_gitlab_host_is_the_reported_error(db: Pool) -> anyhow::Result<()> { +async fn an_unreachable_gitlab_host_is_the_reported_error( + db: Pool, +) -> anyhow::Result<()> { let err = create_repo_webhook( &db, "parent-ws", - "http://glpat-secret@10.0.0.5/grp/proj.git", + "http://glpat-secret@127.0.0.1:1/grp/proj.git", "https://windmill.example/api/w/parent-ws/git_sync/webhook/gitlab", "hook-secret", ) .await - .expect_err("a private host is refused"); + .expect_err("nothing listens on port 1"); assert!( - err.to_string().contains("ALLOW_LOCAL_GIT_REMOTES"), + err.to_string().contains("Could not reach the git host"), "unexpected error: {err}" ); assert!( diff --git a/backend/windmill-common/src/ssrf.rs b/backend/windmill-common/src/ssrf.rs index a16e2d1576..31dee749e6 100644 --- a/backend/windmill-common/src/ssrf.rs +++ b/backend/windmill-common/src/ssrf.rs @@ -8,10 +8,8 @@ pub const ALLOW_PRIVATE_SAML_METADATA_URLS_ENV: &str = "ALLOW_PRIVATE_SAML_METAD pub const ALLOW_PRIVATE_GUEST_JWKS_URLS_ENV: &str = "ALLOW_PRIVATE_GUEST_JWKS_URLS"; -/// Lets Windmill's own git calls reach hosts on a private network. Each process -/// reads it for the calls it makes: the server for remote probes, auto-pull and -/// the GitLab API, the worker that ran a git sync job for the merge request it -/// opens afterwards. So a private git server needs it on both. +/// Lets every git call reach hosts on a private network, whoever it is made for. +/// Without it, [`private_git_host_allowed`] decides. pub const ALLOW_LOCAL_GIT_REMOTES_ENV: &str = "ALLOW_LOCAL_GIT_REMOTES"; /// Why a URL failed SSRF validation. @@ -209,26 +207,52 @@ pub fn allow_private_saml_metadata_urls() -> bool { .is_some_and(|v| v == "true" || v == "1") } -pub fn allow_local_git_remotes() -> bool { +fn allow_local_git_remotes() -> bool { std::env::var(ALLOW_LOCAL_GIT_REMOTES_ENV) .ok() .is_some_and(|v| v == "true" || v == "1") } -/// Appended to a refusal of a private git host, which on a self-hosted -/// instance is usually the organization's own git server rather than an attack. -pub fn local_git_remote_hint() -> String { - format!( - "If your git server is on a private network, set the {ALLOW_LOCAL_GIT_REMOTES_ENV}=true \ - environment variable on the Windmill servers and workers" +/// Who a git call is made for, which decides whether it may reach a host on a +/// private network. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GitRemoteCaller { + /// A workspace admin's request, or Windmill's own work (polling, webhook and + /// token upkeep, the merge request after a deploy), whose errors only admins read. + AdminOrSystem, + /// A request from anyone who is not a workspace admin. + NonAdmin, +} + +/// Whether a git call made for `caller` may reach a host on a private network. +/// +/// The refusal is for non-admins, who may not be able to run code (operators) +/// and would read git's error output back as a probe of the server's network. +/// An admin can run code, which reaches those hosts from a worker already. On a +/// cloud instance, where a workspace admin is anyone who signed up, every caller +/// is refused. +pub fn private_git_host_allowed(caller: GitRemoteCaller) -> bool { + git_host_policy_allows( + caller, + allow_local_git_remotes(), + *crate::worker::CLOUD_HOSTED, ) } -pub fn git_remote_ssrf_error_message(e: &SsrfValidationError) -> String { - match e { - SsrfValidationError::Private { .. } => format!("{e}. {}", local_git_remote_hint()), - _ => e.to_string(), - } +fn git_host_policy_allows(caller: GitRemoteCaller, opted_in: bool, cloud_hosted: bool) -> bool { + opted_in || (caller == GitRemoteCaller::AdminOrSystem && !cloud_hosted) +} + +/// Appended to a refusal of a private git host, naming what would let `caller` +/// through. `None` where nothing an instance administrator sets would help. +pub fn private_git_host_hint(caller: GitRemoteCaller) -> Option { + (caller == GitRemoteCaller::NonAdmin && !*crate::worker::CLOUD_HOSTED).then(|| { + format!( + "Only workspace admins can reach a git server on a private network. To allow \ + every user, set the {ALLOW_LOCAL_GIT_REMOTES_ENV}=true environment variable on \ + the Windmill servers" + ) + }) } pub async fn validate_saml_metadata_url(url: &str) -> Result { @@ -660,6 +684,15 @@ mod tests { )); } + #[test] + fn private_git_hosts_are_refused_to_non_admins_and_on_cloud() { + use GitRemoteCaller::{AdminOrSystem, NonAdmin}; + assert!(git_host_policy_allows(AdminOrSystem, false, false)); + assert!(!git_host_policy_allows(NonAdmin, false, false)); + assert!(!git_host_policy_allows(AdminOrSystem, false, true)); + assert!(git_host_policy_allows(NonAdmin, true, true)); + } + #[tokio::test] async fn saml_ssrf_error_message_includes_env_hint_only_for_private_urls() { let private_error = validate_url_for_ssrf("http://127.0.0.1/metadata") @@ -675,21 +708,4 @@ mod tests { !saml_ssrf_error_message(&invalid_error).contains(ALLOW_PRIVATE_SAML_METADATA_URLS_ENV) ); } - - #[tokio::test] - async fn git_remote_ssrf_error_message_includes_env_hint_only_for_private_urls() { - let private_error = validate_url_for_ssrf("http://10.0.0.5/api/v4") - .await - .unwrap_err(); - assert!( - git_remote_ssrf_error_message(&private_error).contains("ALLOW_LOCAL_GIT_REMOTES=true") - ); - - let invalid_error = validate_url_for_ssrf("gitlab.example.com") - .await - .unwrap_err(); - assert!( - !git_remote_ssrf_error_message(&invalid_error).contains(ALLOW_LOCAL_GIT_REMOTES_ENV) - ); - } } diff --git a/backend/windmill-store/src/resources.rs b/backend/windmill-store/src/resources.rs index c7fdd05c51..2df5dbab5f 100644 --- a/backend/windmill-store/src/resources.rs +++ b/backend/windmill-store/src/resources.rs @@ -17,6 +17,7 @@ use windmill_api_auth::{ }; use windmill_common::db::DB; use windmill_common::per_minute_counter::PerMinuteCounter; +use windmill_common::ssrf::{private_git_host_allowed, private_git_host_hint, GitRemoteCaller}; use windmill_common::workspaces::{check_deploy_rules, RuleCheckResult}; use crate::secret_backend_ext::rename_vault_secret; @@ -3429,8 +3430,10 @@ fn git_url_userinfo(url: &str) -> Option<&str> { git_url_userinfo_range(url).map(|r| &url[r]) } -/// Validates a git URL to prevent option injection, SSRF, and local file read. -async fn validate_git_url(url: &str) -> Result<()> { +/// Validates a git URL to prevent option injection, SSRF, and local file read. The +/// syntax and scheme checks apply to every caller; the private-host refusal only +/// where [`private_git_host_allowed`] refuses `caller`. +async fn validate_git_url(url: &str, caller: GitRemoteCaller) -> Result<()> { let url = url.trim(); if url.is_empty() { return Err(Error::BadRequest("Git URL cannot be empty".to_string())); @@ -3484,17 +3487,17 @@ async fn validate_git_url(url: &str) -> Result<()> { let host = extract_host_from_git_url(url) .ok_or_else(|| Error::BadRequest("Could not parse hostname from git URL".to_string()))?; - // The opt-in for a git server on the instance's own network (and for the CI - // Gitea container on localhost). Scheme and option-injection validation above - // still applies. - if windmill_common::ssrf::allow_local_git_remotes() { + // Scheme and option-injection validation above applies to every caller. + if private_git_host_allowed(caller) { return Ok(()); } - let hint = windmill_common::ssrf::local_git_remote_hint(); + let hint = private_git_host_hint(caller) + .map(|h| format!(" {h}")) + .unwrap_or_default(); if host == "localhost" || host.ends_with(".local") || host == "[::1]" { return Err(Error::BadRequest(format!( - "Git URLs targeting localhost or local network are not allowed. {hint}" + "Git URLs targeting localhost or local network are not allowed.{hint}" ))); } @@ -3502,7 +3505,7 @@ async fn validate_git_url(url: &str) -> Result<()> { if let Ok(ip) = host.parse::() { if is_private_or_reserved_ip(&ip) { return Err(Error::BadRequest(format!( - "Git URLs targeting private or reserved IP addresses are not allowed. {hint}" + "Git URLs targeting private or reserved IP addresses are not allowed.{hint}" ))); } } else { @@ -3525,7 +3528,7 @@ async fn validate_git_url(url: &str) -> Result<()> { for addr in addrs { if is_private_or_reserved_ip(&addr.ip()) { return Err(Error::BadRequest(format!( - "Git URL hostname resolves to a private or reserved IP address. {hint}" + "Git URL hostname resolves to a private or reserved IP address.{hint}" ))); } } @@ -3628,8 +3631,14 @@ async fn get_git_commit_hash( .map_err(|e| { Error::BadRequest(format!("Invalid git repository resource format: {}", e)) })?; + let caller = if authed.is_admin { + GitRemoteCaller::AdminOrSystem + } else { + GitRemoteCaller::NonAdmin + }; git_resource.url = - resolve_azure_devops_url(&db_with_opt_authed, &w_id, &git_resource.url, false).await?; + resolve_azure_devops_url(&db_with_opt_authed, &w_id, &git_resource.url, false, caller) + .await?; // A credential is stored under the repository it was issued for, so a // resource repointed elsewhere finds none. Which credential can be attached // is bounded by that; who may use it is bounded here, on the same terms as @@ -3659,7 +3668,7 @@ async fn get_git_commit_hash( let (git_ssh_cmd, filenames) = get_git_ssh_cmd(&authed, &user_db, &db, &w_id, identities).await?; - let commit_hash = get_repo_latest_commit_hash(&git_resource, git_ssh_cmd).await; + let commit_hash = get_repo_latest_commit_hash(&git_resource, git_ssh_cmd, caller).await; delete_paths(&filenames).await; @@ -3763,12 +3772,17 @@ async fn get_git_ssh_cmd( const GIT_PROBE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60); /// `git` command for a remote probe, with HTTP redirects disabled. `validate_git_url` -/// only vets the host in the URL; git's default (`http.followRedirects=initial`) -/// would let a validated public remote 302 the probe onto a private or link-local -/// address that no check ever sees. Build every probe through this. +/// checks the host in the URL, never one a redirect names; git's default +/// (`http.followRedirects=initial`) would let a public remote 302 the probe of a +/// caller refused private hosts onto one. Build every probe through this. +/// +/// The transports are pinned too: an SCP-shaped remote-helper string such as +/// `ext::@host:path` passes the URL check for a caller allowed private +/// hosts, and only git's own config would stop it from running the command. fn git_probe_command() -> Command { let mut git_cmd = Command::new("git"); git_cmd.args(["-c", "http.followRedirects=false"]); + git_cmd.env("GIT_ALLOW_PROTOCOL", "http:https:ssh:git"); git_cmd } @@ -3828,8 +3842,8 @@ fn dot_git_url(url: &str) -> Option { } /// Run a remote probe, retrying against [`dot_git_url`] if the remote answered the -/// URL as given with a redirect. Extending the path keeps the retry on the host -/// `validate_git_url` already cleared, which is exactly what following the redirect +/// URL as given with a redirect. Extending the path keeps the retry on the host of +/// the URL `validate_git_url` checked, which is exactly what following the redirect /// would not guarantee. `build` must produce the probe for the URL it is handed. /// /// A retry that also fails reports the *original* failure, so the caller's message @@ -3969,6 +3983,7 @@ async fn resolve_azure_devops_url( w_id: &str, url: &str, allow_cache: bool, + caller: GitRemoteCaller, ) -> Result { // Trim first: the http(s) gates the callers apply trim too, so a stored URL with // leading whitespace must not reach the scheme check here as a non-http one. @@ -3981,7 +3996,7 @@ async fn resolve_azure_devops_url( // cost a live credential (nor cache one), and whoever can edit the URL would // otherwise drive a token mint per poll tick. let probe_url = url.replace(placeholder, "windmill"); - validate_git_url(&probe_url).await?; + validate_git_url(&probe_url, caller).await?; // The background poller reads the referenced resource under the system identity, // which bypasses RLS. Confining the destination is what keeps that from becoming an @@ -4181,9 +4196,10 @@ fn git_sync_system_dba(db: &DB) -> DbWithOptAuthed<'static, ApiAuthed> { async fn get_repo_latest_commit_hash( git_resource: &GitRepositoryResource, git_ssh_command: Option, + caller: GitRemoteCaller, ) -> Result { // Validate URL and branch to prevent option injection and SSRF attacks - validate_git_url(&git_resource.url).await?; + validate_git_url(&git_resource.url, caller).await?; let ref_spec = git_resource .branch @@ -4316,8 +4332,14 @@ pub async fn get_git_repo_head_for_autopull( "Automatic pull can't authenticate an SSH git remote in the background. Use an HTTPS URL with an embedded token, or connect the repository through the GitHub App.".to_string(), )); } - git_resource.url = - resolve_azure_devops_url(&git_sync_system_dba(db), w_id, &git_resource.url, true).await?; + git_resource.url = resolve_azure_devops_url( + &git_sync_system_dba(db), + w_id, + &git_resource.url, + true, + GitRemoteCaller::AdminOrSystem, + ) + .await?; // A repo whose credential Windmill holds carries none in its URL, so the // poller has to attach it here or every probe would be unauthenticated. git_resource.url = @@ -4325,14 +4347,15 @@ pub async fn get_git_repo_head_for_autopull( if let Some(branch) = git_resource.branch.as_deref().filter(|s| !s.is_empty()) { let branch = branch.to_string(); - let sha = get_repo_latest_commit_hash(&git_resource, None).await?; + let sha = get_repo_latest_commit_hash(&git_resource, None, GitRemoteCaller::AdminOrSystem) + .await?; return Ok(Some((branch, sha))); } // No explicit branch: resolve the remote's default-branch NAME along with // its head in one call. Fork sync needs the concrete name to scope // `wm-fork//*`, so a bare "HEAD" ref would silently disable it. - validate_git_url(&git_resource.url).await?; + validate_git_url(&git_resource.url, GitRemoteCaller::AdminOrSystem).await?; let output = run_git_probe_for_url(&git_resource.url, "ls-remote --symref HEAD", |url| { let mut git_cmd = git_probe_command(); git_cmd.args(["ls-remote", "--symref", url, "HEAD"]); @@ -4427,13 +4450,20 @@ pub async fn get_git_repo_fork_heads_for_autopull( "Automatic pull can't authenticate an SSH git remote in the background. Use an HTTPS URL with an embedded token, or connect the repository through the GitHub App.".to_string(), )); } - git_resource.url = resolve_azure_devops_url(&dba, w_id, &git_resource.url, true).await?; + git_resource.url = resolve_azure_devops_url( + &dba, + w_id, + &git_resource.url, + true, + GitRemoteCaller::AdminOrSystem, + ) + .await?; // Same reason as the head probe above: a repository whose credential Windmill // holds carries none in its URL, and listing the fork branches is the half of // polling that would otherwise go out unauthenticated. git_resource.url = windmill_common::git_sync_oss::with_stored_credential(db, w_id, git_resource.url).await?; - validate_git_url(&git_resource.url).await?; + validate_git_url(&git_resource.url, GitRemoteCaller::AdminOrSystem).await?; validate_git_ref(base_branch)?; for r in extra_refs { @@ -4819,49 +4849,52 @@ mod tests { )); } + // A caller let through to private hosts must still hit the scheme check. #[tokio::test] async fn test_validate_git_url_blocks_file_scheme() { - let result = validate_git_url("file:///etc/passwd").await; + let result = validate_git_url("file:///etc/passwd", GitRemoteCaller::AdminOrSystem).await; assert!(result.is_err()); assert!(result.unwrap_err().to_string().contains("https://")); } #[tokio::test] async fn test_validate_git_url_blocks_private_ips() { - assert!(validate_git_url("http://127.0.0.1/repo.git").await.is_err()); - assert!(validate_git_url("http://169.254.169.254/latest/meta-data/") - .await - .is_err()); - let err = validate_git_url("http://10.0.0.1/repo.git") - .await - .unwrap_err(); + let v = |url: &'static str| validate_git_url(url, GitRemoteCaller::NonAdmin); + assert!(v("http://127.0.0.1/repo.git").await.is_err()); + assert!(v("http://169.254.169.254/latest/meta-data/").await.is_err()); + let err = v("http://10.0.0.1/repo.git").await.unwrap_err(); assert!(err.to_string().contains("ALLOW_LOCAL_GIT_REMOTES"), "{err}"); - assert!(validate_git_url("http://172.16.0.1/repo.git") - .await - .is_err()); - assert!(validate_git_url("http://192.168.1.1/repo.git") - .await - .is_err()); - assert!(validate_git_url("git://0.0.0.0/repo.git").await.is_err()); + assert!(v("http://172.16.0.1/repo.git").await.is_err()); + assert!(v("http://192.168.1.1/repo.git").await.is_err()); + assert!(v("git://0.0.0.0/repo.git").await.is_err()); // IPv6 loopback, unique-local, and link-local literals - assert!(validate_git_url("git://[::1]/repo.git").await.is_err()); - assert!(validate_git_url("git://[fd00::1]/repo.git").await.is_err()); - assert!(validate_git_url("git://[fe80::1]/repo.git").await.is_err()); + assert!(v("git://[::1]/repo.git").await.is_err()); + assert!(v("git://[fd00::1]/repo.git").await.is_err()); + assert!(v("git://[fe80::1]/repo.git").await.is_err()); + } + + #[tokio::test] + async fn test_validate_git_url_lets_admins_reach_private_hosts() { + assert!( + validate_git_url("http://10.0.0.1/repo.git", GitRemoteCaller::AdminOrSystem) + .await + .is_ok() + ); } #[tokio::test] async fn test_validate_git_url_blocks_localhost() { - assert!(validate_git_url("http://localhost/repo.git").await.is_err()); - assert!(validate_git_url("http://myhost.local/repo.git") - .await - .is_err()); + let v = |url: &'static str| validate_git_url(url, GitRemoteCaller::NonAdmin); + assert!(v("http://localhost/repo.git").await.is_err()); + assert!(v("http://myhost.local/repo.git").await.is_err()); } #[tokio::test] async fn test_validate_git_url_blocks_local_paths() { - assert!(validate_git_url("/etc/passwd").await.is_err()); - assert!(validate_git_url("../relative/path").await.is_err()); - assert!(validate_git_url("./local/repo").await.is_err()); + let v = |url: &'static str| validate_git_url(url, GitRemoteCaller::AdminOrSystem); + assert!(v("/etc/passwd").await.is_err()); + assert!(v("../relative/path").await.is_err()); + assert!(v("./local/repo").await.is_err()); } /// Minimal loopback HTTP server: replies to every request with `response` and @@ -4991,7 +5024,11 @@ mod tests { async fn test_validate_git_url_fails_closed_on_unresolvable_host() { // `.invalid` never resolves (RFC 6761). The private-IP check is only // meaningful if a failed lookup rejects instead of falling through. - let result = validate_git_url("https://this-host-does-not-exist.invalid/repo.git").await; + let result = validate_git_url( + "https://this-host-does-not-exist.invalid/repo.git", + GitRemoteCaller::NonAdmin, + ) + .await; assert!( result.is_err(), "an unresolvable host was allowed — does this resolver synthesize records for NXDOMAIN?" @@ -5002,21 +5039,33 @@ mod tests { #[tokio::test] async fn test_validate_git_url_allows_valid_urls() { // Needs DNS: validation fails closed on a host it cannot resolve. - assert!(validate_git_url("https://github.com/user/repo.git") + let v = |url: &'static str| validate_git_url(url, GitRemoteCaller::NonAdmin); + assert!(v("https://github.com/user/repo.git").await.is_ok()); + assert!(v("git@github.com:user/repo.git").await.is_ok()); + assert!(v("ssh://git@github.com/user/repo.git").await.is_ok()); + } + + #[tokio::test] + async fn test_git_probe_refuses_remote_helpers() { + // A caller allowed private hosts skips the DNS step that would reject this + // SCP-shaped string, so the transport pin is what keeps git from running it. + let output = git_probe_command() + .args(["ls-remote", "testhelper::x@127.0.0.1:repo"]) + .output() .await - .is_ok()); - assert!(validate_git_url("git@github.com:user/repo.git") - .await - .is_ok()); - assert!(validate_git_url("ssh://git@github.com/user/repo.git") - .await - .is_ok()); + .unwrap(); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("transport 'testhelper' not allowed"), + "{stderr}" + ); } #[tokio::test] async fn test_validate_git_url_blocks_option_injection() { - assert!(validate_git_url("-evil").await.is_err()); - assert!(validate_git_url("--upload-pack=evil").await.is_err()); + let v = |url: &'static str| validate_git_url(url, GitRemoteCaller::AdminOrSystem); + assert!(v("-evil").await.is_err()); + assert!(v("--upload-pack=evil").await.is_err()); } #[test] @@ -5112,24 +5161,21 @@ mod tests { // GHSA-p5cj-8cfh-mjv6: a loopback authority must stay blocked, and the // fragment/query `@public-host` bypasses of #8600 must be rejected so the // host git dials can never diverge from the validated host. - assert!(validate_git_url("http://127.0.0.1:40173/repo.git") - .await - .is_err()); - assert!(validate_git_url( - "http://127.0.0.1:40173/repo.git#@github.com/windmill-labs/windmill.git" - ) - .await - .is_err()); - assert!(validate_git_url( - "http://127.0.0.1:40173/repo.git?@github.com/windmill-labs/windmill.git" - ) - .await - .is_err()); - // A legitimate public repo URL still validates. + let v = |url: &'static str| validate_git_url(url, GitRemoteCaller::NonAdmin); + assert!(v("http://127.0.0.1:40173/repo.git").await.is_err()); assert!( - validate_git_url("https://github.com/windmill-labs/windmill.git") + v("http://127.0.0.1:40173/repo.git#@github.com/windmill-labs/windmill.git") .await - .is_ok() + .is_err() ); + assert!( + v("http://127.0.0.1:40173/repo.git?@github.com/windmill-labs/windmill.git") + .await + .is_err() + ); + // A legitimate public repo URL still validates. + assert!(v("https://github.com/windmill-labs/windmill.git") + .await + .is_ok()); } } From f8f7c0009f32c1440566420725b12e78ca804b03 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 11 Sep 2026 10:42:29 +0200 Subject: [PATCH 076/144] fix: serve instance env settings at the documented /settings/local path (#11075) Claude-Session: https://claude.ai/code/session_01RbLzVfFDZ9pjBGHSzCSrNZ Co-authored-by: Claude Opus 5 (1M context) --- backend/windmill-api-settings/src/lib.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/backend/windmill-api-settings/src/lib.rs b/backend/windmill-api-settings/src/lib.rs index 81528912a0..e7be0d5ede 100644 --- a/backend/windmill-api-settings/src/lib.rs +++ b/backend/windmill-api-settings/src/lib.rs @@ -113,6 +113,9 @@ async fn get_ruff_config_unauthed(Extension(db): Extension) -> error::Result pub fn global_service() -> Router { #[warn(unused_mut)] let r = Router::new() + // `/local` is the path in openapi.yaml, so every generated client (getLocal) calls it; + // `/envs` stays for callers that found the route in the code. + .route("/local", get(get_local_settings)) .route("/envs", get(get_local_settings)) .route( "/global/{key}", From 57f8b0826ad61cb118d0cafbbc9203327d858940 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Fri, 11 Sep 2026 10:48:02 +0200 Subject: [PATCH 077/144] fix: keep pinned import versions of imported scripts in bun lockfiles (#11082) * fix: keep pinned import versions of imported scripts in bun lockfiles Co-Authored-By: Claude Opus 5 (1M context) * test: guard pinned imports through an unlocked multi-file bun run Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Opus 5 (1M context) --- backend/tests/script_modules.rs | 55 +++++++++++++++++++++++++++ backend/windmill-worker/loader.bun.js | 4 +- 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/backend/tests/script_modules.rs b/backend/tests/script_modules.rs index 8755ebfb57..eec9d3b45a 100644 --- a/backend/tests/script_modules.rs +++ b/backend/tests/script_modules.rs @@ -159,3 +159,58 @@ export function main(name: string) { assert_eq!(result, json!("hello world")); Ok(()) } + +/// A multi-file script run without a lock is bundled by the lockfile build. A pinned import in +/// a workspace script it imports must be installed at that version and still resolve at run time. +#[sqlx::test(fixtures("base"))] +async fn test_bun_module_imports_pinned_workspace_script(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + sqlx::query( + "INSERT INTO script (workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) + VALUES ('test-workspace', 'test-user', $1, '{}', '', '', 'f/system/pinned_module', 12350, 'bun', '')", + ) + .bind( + r#" +import _ from "lodash@4.17.20"; +export function lodashVersion() { return _.VERSION; } +"#, + ) + .execute(&db) + .await?; + + let mut modules = HashMap::new(); + modules.insert( + "helper.ts".to_string(), + ScriptModule { + content: "export function label(v: string) { return v; }\n".to_string(), + language: ScriptLang::Bun, + lock: None, + }, + ); + + let job = JobPayload::Code(RawCode { + content: r#" +import { lodashVersion } from "/f/system/pinned_module"; +import { label } from "./helper.ts"; +export function main() { return label(lodashVersion()); } +"# + .to_owned(), + path: Some("f/system/my_script".to_string()), + language: ScriptLang::Bun, + modules: Some(modules), + tag: None, + ..RawCode::default() + }); + + let result = RunJob::from(job) + .run_until_complete(&db, false, port) + .await + .json_result() + .unwrap(); + + assert_eq!(result, json!("4.17.20")); + Ok(()) +} diff --git a/backend/windmill-worker/loader.bun.js b/backend/windmill-worker/loader.bun.js index d03f19b6ba..9e0e071cca 100644 --- a/backend/windmill-worker/loader.bun.js +++ b/backend/windmill-worker/loader.bun.js @@ -106,8 +106,8 @@ const p = { const hash = TEMP_SCRIPT_REFS?.[normalized]; const url = (isRelative - ? `${base_internal_url}/api/w/${w_id}/scripts/raw_unpinned/p/${file_path}/../${args.path}${endExt}` - : `${base_internal_url}/api/w/${w_id}/scripts/raw_unpinned/p/${args.path}${endExt}` + ? `${base_internal_url}/api/w/${w_id}/scripts/RAW_GET_ENDPOINT/p/${file_path}/../${args.path}${endExt}` + : `${base_internal_url}/api/w/${w_id}/scripts/RAW_GET_ENDPOINT/p/${args.path}${endExt}` ) + (hash ? `?temp_script_hash=${hash}` : ""); const file = isRelative ? resolve("./" + file_path + "/../" + args.path + ".url") From 172d6c275b92b39d13848b543df9db10148e94ef Mon Sep 17 00:00:00 2001 From: AlexRV12 <71396855+AlexRV12@users.noreply.github.com> Date: Fri, 11 Sep 2026 10:53:39 +0200 Subject: [PATCH 078/144] feat: run a flow test through the chat's argument form (#11069) * feat: run a flow test through the chat's argument form Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01MxWRaAbciaeRxam5gPzaaV * fix: resolve the flow editor to run when the form is submitted Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01MxWRaAbciaeRxam5gPzaaV * refactor: trim the flow run form's duplication and narration Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01MxWRaAbciaeRxam5gPzaaV * fix: label test_run_flow in the bypassed-tools tooltip Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01MxWRaAbciaeRxam5gPzaaV * fix: show the flow icon on a flow's run form in the preview panel Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01MxWRaAbciaeRxam5gPzaaV --------- Co-authored-by: Claude Opus 5 (1M context) --- .../copilot/chat/RunArgsFormDisplay.svelte | 14 +- .../copilot/chat/RunScriptCard.svelte | 9 +- .../copilot/chat/global/core.test.ts | 125 +++++++++++++++++- .../components/copilot/chat/global/core.ts | 99 +++++++------- .../src/lib/components/copilot/chat/shared.ts | 10 +- .../sessions/RunFormPreviewSlot.svelte | 6 +- 6 files changed, 191 insertions(+), 72 deletions(-) diff --git a/frontend/src/lib/components/copilot/chat/RunArgsFormDisplay.svelte b/frontend/src/lib/components/copilot/chat/RunArgsFormDisplay.svelte index afa9df0c25..6072b6e341 100644 --- a/frontend/src/lib/components/copilot/chat/RunArgsFormDisplay.svelte +++ b/frontend/src/lib/components/copilot/chat/RunArgsFormDisplay.svelte @@ -37,6 +37,10 @@ // it would discard those edits. const draft = untrack(() => aiChatManager.runFormDraft(toolCallId, runForm)) + const runnableKind = $derived(runForm.runnableKind ?? 'script') + const staleFormToast = () => + sendUserToast(`This run form is no longer active — ask again to run the ${runnableKind}.`, true) + const properties = $derived(draft.schema?.properties ?? {}) const hasArgs = $derived(Object.keys(properties).length > 0) @@ -71,7 +75,7 @@ // manager that opened it, so submitting would mint an ephemeral secret variable // per click and still run nothing. if (!aiChatManager.isRunFormPending(toolCallId)) { - sendUserToast('This run form is no longer active — ask again to run the script.', true) + staleFormToast() return } // Ahead of processSecretArgs, which writes ephemeral variables to the workspace: the @@ -102,7 +106,7 @@ // then the ephemeral variables exist — say so rather than leaving a dead button. if (!aiChatManager.handleRunFormSubmit(toolCallId, processed)) { aiChatManager.endRunFormSubmit(toolCallId) - sendUserToast('This run form is no longer active — ask again to run the script.', true) + staleFormToast() } } @@ -151,7 +155,7 @@ ? runForm.code && runForm.lang ? { source: 'inline', code: runForm.code, lang: runForm.lang } : undefined - : { source: 'deployed', path: runForm.path, runnable_kind: 'script' }} + : { source: 'deployed', path: runForm.path, runnable_kind: runnableKind }} disabled={planMode} {workspace} prettifyHeader @@ -159,7 +163,7 @@ bind:args={draft.args} /> {:else} -

    This script takes no arguments.

    +

    This {runnableKind} takes no arguments.

    {/if}
    @@ -196,7 +200,7 @@ {/if} {#if runForm.resetKeys?.length}

    - Disabled by this script, so it will run with its default: + Disabled by this {runnableKind}, so it will run with its default: {runForm.resetKeys.join(', ')}

    {/if} diff --git a/frontend/src/lib/components/copilot/chat/RunScriptCard.svelte b/frontend/src/lib/components/copilot/chat/RunScriptCard.svelte index 4d19ce2ecf..edd1704bb2 100644 --- a/frontend/src/lib/components/copilot/chat/RunScriptCard.svelte +++ b/frontend/src/lib/components/copilot/chat/RunScriptCard.svelte @@ -27,6 +27,7 @@ let { message }: Props = $props() const runForm = $derived(message.runForm!) + const runnableKind = $derived(runForm.runnableKind ?? 'script') // The loop is parked on the form and nothing has run yet: the card is the form. const pending = $derived(isActiveRunForm(message)) @@ -103,8 +104,8 @@ ) const cancelReason = $derived( ran - ? 'This run was cancelled while the script was running.' - : 'This run was cancelled before the script started.' + ? `This run was cancelled while the ${runnableKind} was running.` + : `This run was cancelled before the ${runnableKind} started.` ) // Streaming opens the tab early: the result is already arriving, and one that appeared // only at the end would hide the thing the user is waiting to read. @@ -296,7 +297,7 @@ tab it already opened. The row's only control, as on every other tool call. --> {#snippet previewChip()} aiChatManager.cancelJob(chatJob.jobId)} > Cancel diff --git a/frontend/src/lib/components/copilot/chat/global/core.test.ts b/frontend/src/lib/components/copilot/chat/global/core.test.ts index 1132c4c77f..bcf7bfb6e2 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.test.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.test.ts @@ -4750,12 +4750,17 @@ describe('global AI tools', () => { ) }) + // The form offers the arguments the flow declares and no others, so a fixture flow that + // takes one has to say so — as a real flow does, since nothing else could render a field. + const FLOW_NAME_SCHEMA = { type: 'object', properties: { name: { type: 'string' } } } + it('test_run_flow previews draft flow content by path', async () => { const modules = [{ id: 'start', value: { type: 'identity' } }] await callGlobalTool('write_flow', { path: 'f/flows/draft-test', summary: 'Draft test flow', - modules: JSON.stringify(modules) + modules: JSON.stringify(modules), + schema: JSON.stringify(FLOW_NAME_SCHEMA) }) await withCompletedTestJob(() => @@ -4782,7 +4787,7 @@ describe('global AI tools', () => { path: 'f/flows/deployed-test', summary: 'Deployed test flow', value: { modules }, - schema: {} + schema: FLOW_NAME_SCHEMA } as any) await withCompletedTestJob(() => @@ -4816,7 +4821,7 @@ describe('global AI tools', () => { path: 'u/admin/live_flow', summary: 'Live flow', value: { modules: [{ id: 'live_step', value: { type: 'identity' } }] }, - schema: {}, + schema: FLOW_NAME_SCHEMA, edited_by: '', edited_at: '', archived: false, @@ -4839,12 +4844,14 @@ describe('global AI tools', () => { path: 'u/admin/live_flow', args: { name: 'Ada' } }, - toolCallbacks, + { ...toolCallbacks, requestRunArgs: async () => ({ name: 'Grace' }) }, { testActiveFlow } ) ) - expect(testActiveFlow).toHaveBeenCalledWith('u/admin/live_flow_storage', { name: 'Ada' }) + // What the form submitted, not what the model proposed: the editor runs the flow, but + // the arguments are the user's. + expect(testActiveFlow).toHaveBeenCalledWith('u/admin/live_flow_storage', { name: 'Grace' }) expect(FlowService.getFlowByPath).not.toHaveBeenCalled() expect(JobService.runFlowPreview).not.toHaveBeenCalled() expect(result).toContain('Result (SUCCESS)') @@ -4858,7 +4865,7 @@ describe('global AI tools', () => { path: 'u/admin/live_flow_fallback', summary: 'Live flow fallback', value: { modules: [{ id: 'fallback_step', value: { type: 'identity' } }] }, - schema: {}, + schema: FLOW_NAME_SCHEMA, edited_by: '', edited_at: '', archived: false, @@ -4898,6 +4905,110 @@ describe('global AI tools', () => { }) }) + // A flow reaches its run form the way a script does: opened on the flow's own input schema, + // with the dynamic-option picker the schema carries, and the run takes what came back. + it('test_run_flow opens the form on the flow schema and runs what it submitted', async () => { + const modules = [{ id: 'start', value: { type: 'identity' } }] + await callGlobalTool('write_flow', { + path: 'f/flows/formed-flow', + summary: 'Formed flow', + modules: JSON.stringify(modules), + schema: JSON.stringify({ + ...FLOW_NAME_SCHEMA, + 'x-windmill-dyn-select-code': 'export function names() { return ["Ada"] }', + 'x-windmill-dyn-select-lang': 'bun' + }) + }) + + let form: any + await withCompletedTestJob(() => + callGlobalTool( + 'test_run_flow', + { path: 'f/flows/formed-flow', args: { name: 'Ada' } }, + { + ...toolCallbacks, + requestRunArgs: async (_toolId, f) => { + form = f + return { name: 'Grace' } + } + } + ) + ) + + expect(form.args).toEqual({ name: 'Ada' }) + expect(form.runnableKind).toBe('flow') + expect(form.schema?.properties).toEqual(FLOW_NAME_SCHEMA.properties) + // The flow's own dynselect script, which the schema carries rather than a step. + expect({ code: form.code, lang: form.lang }).toEqual({ + code: 'export function names() { return ["Ada"] }', + lang: 'bun' + }) + expect(JobService.runFlowPreview).toHaveBeenCalledWith({ + workspace: WORKSPACE, + requestBody: { + path: 'f/flows/formed-flow', + value: { modules }, + args: { name: 'Grace' } + } + }) + }) + + // The form waits as long as the user does, so which editor is on screen is only known when + // they press Run — checking it when the card appeared would drive an editor they have since + // moved away from. + it('test_run_flow re-checks the editor on screen when the form is submitted', async () => { + const modules = [{ id: 'moved_step', value: { type: 'identity' } }] + seedBackendDraft( + 'flow', + 'u/admin/moved_flow', + { + path: 'u/admin/moved_flow', + summary: 'Moved flow', + value: { modules }, + schema: FLOW_NAME_SCHEMA, + edited_by: '', + edited_at: '', + archived: false, + extra_perms: {} + }, + { workspace: WORKSPACE } + ) + UserDraft.setLiveEditorDraft({ + workspace: WORKSPACE, + itemKind: 'flow', + storagePath: 'u/admin/moved_flow', + effectivePath: 'u/admin/moved_flow' + }) + const testActiveFlow = vi.fn(async () => 'job-live-flow') + + await withCompletedTestJob(() => + callGlobalTool( + 'test_run_flow', + { path: 'u/admin/moved_flow', args: { name: 'Ada' } }, + { + ...toolCallbacks, + requestRunArgs: async (_toolId, form) => { + // The preview panel moves to another flow while the form sits open. + UserDraft.setLiveEditorDraft({ + workspace: WORKSPACE, + itemKind: 'flow', + storagePath: 'u/admin/other_flow', + effectivePath: 'u/admin/other_flow' + }) + return form.args + } + }, + { testActiveFlow } + ) + ) + + expect(testActiveFlow).not.toHaveBeenCalled() + expect(JobService.runFlowPreview).toHaveBeenCalledWith({ + workspace: WORKSPACE, + requestBody: { path: 'u/admin/moved_flow', value: { modules }, args: { name: 'Ada' } } + }) + }) + // The flow may be open in a session tab that isn't the one on screen: driving its editor // would paint the run into a tab the user is not looking at. it('test_run_flow previews rather than driving an editor the user is not looking at', async () => { @@ -4908,7 +5019,7 @@ describe('global AI tools', () => { path: 'u/admin/background_flow', summary: 'Background flow', value: { modules: [{ id: 'background_step', value: { type: 'identity' } }] }, - schema: {}, + schema: FLOW_NAME_SCHEMA, edited_by: '', edited_at: '', archived: false, diff --git a/frontend/src/lib/components/copilot/chat/global/core.ts b/frontend/src/lib/components/copilot/chat/global/core.ts index 868d0e1728..0fe23361e7 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.ts @@ -929,7 +929,7 @@ const testRunFlowSchema = z.object({ const testRunFlowToolDef = createToolDef( testRunFlowSchema, 'test_run_flow', - 'Execute a preview-style test run of a flow by path, preferring draft content when it exists.', + 'Execute a preview-style test run of a flow by path, preferring draft content when it exists. The user gets an argument form prefilled with `args` and may edit or dismiss it before it runs, so fill in every argument you can infer. For a secret argument prefer `$var:` naming an existing workspace variable; a literal is minted into a short-lived secret before the run, but stays in this call.', { strict: false } ) @@ -1348,7 +1348,7 @@ ${pipelineBullet} : ' Pass items (":" entries naming the items you changed) so the review is scoped to them — omitting items preselects every pending change in the workspace' }, or mode ("draft" or "fork") to force which comparison is shown. Prefer offering this review page over calling deploy_workspace_item directly when several items changed. - For a Windmill operation no other tool covers (workers, queue state, a run's args, ...), use search_api_endpoints to find a REST endpoint, then call_api_get for reads or call_api_endpoint for mutations (the user is asked to confirm those). Always prefer a dedicated tool when one exists; endpoints for authoring or deleting scripts, flows, apps, schedules, resources, or variables are not available through the API catalog tools — use the draft tools and delete_workspace_item instead. -- Default to test_run_script, test_run_flow, or test_run_step for any run request, an existing script included; they prefer drafts and need no deployment. Use run_script only when the user names the deployed version ("the deployed X", "in production", "for real") — a bare "run X" is not that. For run_script, read the item with read_workspace_item version: "deployed" first so the arguments match the deployed schema, and fill in every one you can infer. runFlowByPath from the API catalog runs a deployed flow without a form: only for a flow the user asked to run deployed. +- Default to test_run_script, test_run_flow, or test_run_step for any run request, an existing script included; they prefer drafts and need no deployment. Use run_script only when the user names the deployed version ("the deployed X", "in production", "for real") — a bare "run X" is not that. For run_script, read the item with read_workspace_item version: "deployed" first so the arguments match the deployed schema, and fill in every one you can infer. test_run_script, run_script and test_run_flow all show the user an argument form prefilled with what you sent, so fill in every argument you can infer rather than asking for it in chat. runFlowByPath from the API catalog is the exception — it runs a deployed flow with no form at all: only for a flow the user asked to run deployed. - When a required decision is ambiguous, use askUserQuestion with two to ten clear proposed answer strings instead of guessing. The user can also type a custom answer when none of the proposed answers fit. Set multiSelect: true only when the answers can genuinely co-apply and the user may pick several (not mutually exclusive). - When the user asks you to remember a lasting preference, always/never do something, or change/stop a behavior going forward, call update_user_instructions to persist it. It edits only the USER INSTRUCTIONS block (not WORKSPACE INSTRUCTIONS). Keep each instruction concise; do not use it for one-off requests scoped to the current task. - Keep context targeted.${ @@ -3696,8 +3696,10 @@ export const globalTools: Tool<{}>[] = [ const parsed = testRunFlowSchema.parse(ctx.args) return testRunFlowByPath(parsed, ctx) }, - requiresConfirmation: true, - confirmationMessage: (args) => `Run a test of ${pathLeaf(args?.path, 'the flow')}`, + // No requiresConfirmation, for the reason test_run_script carries. + bypassedByAutoAccept: true, + streamingLabel: 'Preparing the test form...', + confirmationMessage: 'Run a test of a flow', queuedLabel: (args) => `Test ${args?.path ?? 'the flow'}`, showDetails: true, autoCollapseDetails: false @@ -5501,8 +5503,8 @@ async function testRunScriptByPath( /** The "do not call again" half is load-bearing: without it the model re-proposes the * call, which re-opens the form the user just dismissed, and Stop becomes their only * way out. */ -const runFormCancelled = (toolName: string) => - `The user cancelled the run form. The script did NOT run. Do not call ${toolName} again unless the user asks for it.` +const runFormCancelled = (toolName: string, noun: string) => + `The user cancelled the run form. The ${noun} did NOT run. Do not call ${toolName} again unless the user asks for it.` /** The model only needs to see what the user changed, and nothing bounds an object or * array argument the form let them paste into. */ @@ -5550,7 +5552,7 @@ async function runThroughForm(spec: FormRunSpec, ctx: WriteDraftCtx): Promise, ctx: WriteDraftCtx ): Promise { - const { workspace, toolId, toolCallbacks } = ctx - const testArgs = normalizeTestRunArgs(args.args) - const testActiveFlow = liveFlowTestHookFromCtx(ctx, args.path) + const { workspace } = ctx + // The schema must be in hand before the form is built, and the value rides along from the + // same read so the fields and the previewed flow are one version. With an editor open on + // this path this reads its in-memory cell rather than the network. + const flow = await loadFlowDraftValue(args.path, workspace) + const schema = (flow.flow.schema as Record | null | undefined) ?? {} - if (testActiveFlow) { - return executeTestRun({ - jobStarter: async () => { - const jobId = await testActiveFlow(testArgs) + return runThroughForm( + { + path: args.path, + schema, + summary: flow.summary, + kind: 'test', + // A flow's dynamic-option pickers are one script stored on the schema itself, which is + // where the editor's own test form reads them from (FlowPreviewContent). + code: schema['x-windmill-dyn-select-code'], + lang: schema['x-windmill-dyn-select-lang'], + // Never "deployed": a test run previews the draft, so a line sending the model to the + // deployed schema would name the wrong version. + schemaNoun: 'flow', + toolName: 'test_run_flow', + proposed: args.args, + startMessage: `Starting flow test run for "${args.path}"...`, + contextName: 'flow', + // The model is told to test and iterate, so the bypass posture answers the form. + autoAcceptable: true, + background: args.background, + detachAfterMs: waitSecondsToDetachMs(args.wait_seconds), + startJob: async (submitted) => { + // An open editor runs its own in-memory flow and paints the run in its graph. + // Resolved here rather than before the form: the form waits as long as the user + // does, and the editor on screen when they press Run is the one it belongs in. + const jobId = await liveFlowTestHookFromCtx(ctx, args.path)?.(submitted) if (jobId) { return jobId } - - const flow = await loadFlowDraftValue(args.path, workspace) return JobService.runFlowPreview({ workspace, requestBody: { path: args.path, value: flowDraftValueForPreview(flow.flow), - args: testArgs + args: submitted } }) - }, - workspace, - toolCallbacks, - toolId, - startMessage: `Starting flow test run for "${args.path}"...`, - contextName: 'flow', - background: args.background, - detachAfterMs: waitSecondsToDetachMs(args.wait_seconds), - label: args.path - }) - } - - const flow = await loadFlowDraftValue(args.path, workspace) - - return executeTestRun({ - jobStarter: () => - JobService.runFlowPreview({ - workspace, - requestBody: { - path: args.path, - value: flowDraftValueForPreview(flow.flow), - args: testArgs - } - }), - workspace, - toolCallbacks, - toolId, - startMessage: `Starting flow test run for "${args.path}"...`, - contextName: 'flow', - background: args.background, - detachAfterMs: waitSecondsToDetachMs(args.wait_seconds), - label: args.path - }) + } + }, + ctx + ) } async function testRunFlowStepByPath( diff --git a/frontend/src/lib/components/copilot/chat/shared.ts b/frontend/src/lib/components/copilot/chat/shared.ts index bc429d9d15..adb0f63be4 100644 --- a/frontend/src/lib/components/copilot/chat/shared.ts +++ b/frontend/src/lib/components/copilot/chat/shared.ts @@ -561,14 +561,18 @@ export type RunFormDisplay = { /** What the run is, in the card's own words: a deployed script run, or a preview of the * draft being written. Only the tense of the row's label turns on it. */ kind?: 'run' | 'test' + /** What is being run, for the noun the card says it in. + * Absent on cards recorded before flows had a form, which were all scripts. */ + runnableKind?: 'script' | 'flow' /** Of whatever version is about to run: the deployed script, or the draft a test * previews. Only the rendered form reads it, so it is dropped once one of the flags * below unmounts that form: kept, every settled card would carry a copy of the schema * — password and file defaults included — in history forever. */ schema?: Record - /** The draft a test run previews, for the `dynselect-` helper only — a deployed helper - * would answer for the wrong version. Set on a test run alone, and dropped with the - * schema once the form unmounts, so no settled card carries a copy of the code. */ + /** The script the `dynselect-` helper runs: the draft a script test run previews, since a + * deployed helper would answer for the wrong version, or the one a flow schema carries on + * itself. Dropped with the schema once the form unmounts, so no settled card keeps a copy + * of the code. */ code?: string lang?: ScriptLang /** Prefill only: the card's `parameters` records what the job started with. */ diff --git a/frontend/src/lib/components/sessions/RunFormPreviewSlot.svelte b/frontend/src/lib/components/sessions/RunFormPreviewSlot.svelte index 336c03fb1a..3384086da7 100644 --- a/frontend/src/lib/components/sessions/RunFormPreviewSlot.svelte +++ b/frontend/src/lib/components/sessions/RunFormPreviewSlot.svelte @@ -1,6 +1,6 @@ - + +
    {#if readOnly} From e651b4cd63c3bd64a8739c9f60c6dfa437f19b4b Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 11 Sep 2026 13:14:09 +0200 Subject: [PATCH 080/144] perf: lazy-load the low-code runtime on public app pages (#11087) * perf: lazy-load the low-code runtime on public app pages Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_018bSzETfUb23CRRMEJdwqeS * perf: fetch the low-code runtime alongside the app payload Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_018bSzETfUb23CRRMEJdwqeS --------- Co-authored-by: Claude Opus 5 (1M context) --- frontend/src/lib/components/DiffDrawer.svelte | 2 +- frontend/src/lib/components/FlowViewer.svelte | 3 +- .../lib/components/LocalDraftBanner.svelte | 8 +-- frontend/src/lib/components/Login.svelte | 16 +++--- .../components/WorkspaceItemDiffViewer.svelte | 3 +- .../apps/editor/DeploymentHistory.svelte | 2 +- .../components/apps/editor/PublicApp.svelte | 57 +++++++++++-------- .../apps/editor/PublicAppFrame.svelte | 9 ++- .../components/apps/editor/loadAppPreview.ts | 12 ++++ .../components/raw_apps/rawAppDiffUtils.ts | 3 +- frontend/src/lib/navigation.ts | 2 +- frontend/src/lib/sharedUtils.ts | 5 +- frontend/src/lib/svelte5UtilsKit.svelte.ts | 9 +-- frontend/src/lib/utils.ts | 27 +-------- frontend/src/lib/utils/capitalize.ts | 3 + frontend/src/lib/utils/orderedYaml.ts | 23 ++++++++ frontend/src/lib/utils/serializeParam.ts | 12 ++++ frontend/vite.config.js | 49 +++++++++++++++- 18 files changed, 161 insertions(+), 84 deletions(-) create mode 100644 frontend/src/lib/components/apps/editor/loadAppPreview.ts create mode 100644 frontend/src/lib/utils/capitalize.ts create mode 100644 frontend/src/lib/utils/orderedYaml.ts create mode 100644 frontend/src/lib/utils/serializeParam.ts diff --git a/frontend/src/lib/components/DiffDrawer.svelte b/frontend/src/lib/components/DiffDrawer.svelte index 25e94a8efe..d70046d62a 100644 --- a/frontend/src/lib/components/DiffDrawer.svelte +++ b/frontend/src/lib/components/DiffDrawer.svelte @@ -7,10 +7,10 @@ import { cleanValueProperties, orderedJsonStringify, - orderedYamlStringify, replaceFalseWithUndefined, type Value } from '$lib/utils' + import { orderedYamlStringify } from '$lib/utils/orderedYaml' import type { Script } from '$lib/gen' type DiffData = { diff --git a/frontend/src/lib/components/FlowViewer.svelte b/frontend/src/lib/components/FlowViewer.svelte index 1e321584e3..03baebb501 100644 --- a/frontend/src/lib/components/FlowViewer.svelte +++ b/frontend/src/lib/components/FlowViewer.svelte @@ -5,7 +5,8 @@ import SchemaViewer from './SchemaViewer.svelte' import FlowGraphViewer from './FlowGraphViewer.svelte' import { Loader2 } from 'lucide-svelte' - import { orderedYamlStringify, cleanValueProperties, replaceFalseWithUndefined } from '$lib/utils' + import { cleanValueProperties, replaceFalseWithUndefined } from '$lib/utils' + import { orderedYamlStringify } from '$lib/utils/orderedYaml' import { workspaceStore } from '$lib/stores' import { watch } from 'runed' diff --git a/frontend/src/lib/components/LocalDraftBanner.svelte b/frontend/src/lib/components/LocalDraftBanner.svelte index dff131b281..af2787222e 100644 --- a/frontend/src/lib/components/LocalDraftBanner.svelte +++ b/frontend/src/lib/components/LocalDraftBanner.svelte @@ -2,12 +2,8 @@ import { Button } from '$lib/components/common' import DiffDrawer from '$lib/components/DiffDrawer.svelte' import { classes } from '$lib/components/common/alert/model' - import { - cleanValueProperties, - orderedYamlStringify, - replaceFalseWithUndefined, - type Value - } from '$lib/utils' + import { cleanValueProperties, replaceFalseWithUndefined, type Value } from '$lib/utils' + import { orderedYamlStringify } from '$lib/utils/orderedYaml' import { AlertCircle, Diff } from 'lucide-svelte' import { twMerge } from 'tailwind-merge' import { fade } from 'svelte/transition' diff --git a/frontend/src/lib/components/Login.svelte b/frontend/src/lib/components/Login.svelte index 0307643a08..e4b73478aa 100644 --- a/frontend/src/lib/components/Login.svelte +++ b/frontend/src/lib/components/Login.svelte @@ -16,15 +16,13 @@ @@ -116,20 +111,6 @@ >Powered by   Windmill
    - - {#snippet userInfo(child)} -
    {child}
    - {/snippet} - - {#if showLoginStatus} -
    {#if $userStore} - {@render userInfo($userStore.username)} - {:else if globalUser} - {@render userInfo(globalUser.email)} - {:else}{/if} -
    - {/if} {/if} {#if notExists} diff --git a/frontend/src/lib/mcpEndpointTools.ts b/frontend/src/lib/mcpEndpointTools.ts index d3da430240..1398e8dfa9 100644 --- a/frontend/src/lib/mcpEndpointTools.ts +++ b/frontend/src/lib/mcpEndpointTools.ts @@ -1290,10 +1290,6 @@ export const mcpEndpointTools: EndpointTool[] = [ "type": "string" }, "description": "Raw apps: author-declared scopes for the frontend SDK token. Takes effect only when `sandbox` is also true \u2014 an unsandboxed bundle runs with the viewer's own session, so no token is advertised or minted for it and this list stays inert. On a sandboxed app a non-empty list lets viewers mint (after consenting) a short-lived token carrying their own identity restricted to these scopes, handed to the app bundle so `windmill-client` calls run as the viewer. Must be a subset of the server's curated allowlist (jobs:run, jobs:read, users:read, resources:read, variables:read).\n" - }, - "hide_login_status": { - "type": "boolean", - "description": "When true, the app's public and custom URLs do not show the viewer's login status (the user they are signed in as, or that they are signed out) in the top-left corner. Absent or false shows it.\n" } } } @@ -1409,10 +1405,6 @@ export const mcpEndpointTools: EndpointTool[] = [ "type": "string" }, "description": "Raw apps: author-declared scopes for the frontend SDK token. Takes effect only when `sandbox` is also true \u2014 an unsandboxed bundle runs with the viewer's own session, so no token is advertised or minted for it and this list stays inert. On a sandboxed app a non-empty list lets viewers mint (after consenting) a short-lived token carrying their own identity restricted to these scopes, handed to the app bundle so `windmill-client` calls run as the viewer. Must be a subset of the server's curated allowlist (jobs:run, jobs:read, users:read, resources:read, variables:read).\n" - }, - "hide_login_status": { - "type": "boolean", - "description": "When true, the app's public and custom URLs do not show the viewer's login status (the user they are signed in as, or that they are signed out) in the top-left corner. Absent or false shows it.\n" } } }, From d539e8674f2bf92d6c10b182ed4a1073714062c0 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 11 Sep 2026 14:01:20 +0200 Subject: [PATCH 083/144] fix(dbt): stop dbt sending anonymous usage stats from workers (#11091) Claude-Session: https://claude.ai/code/session_01KJZKmyRJZWUmSS7H1vV6Hs Co-authored-by: Claude Opus 5 (1M context) --- backend/windmill-worker/src/dbt_executor.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/backend/windmill-worker/src/dbt_executor.rs b/backend/windmill-worker/src/dbt_executor.rs index bb449f1810..413448b644 100644 --- a/backend/windmill-worker/src/dbt_executor.rs +++ b/backend/windmill-worker/src/dbt_executor.rs @@ -2271,7 +2271,11 @@ pub(crate) fn dbt_command(p: &PreparedProject, args: &[&str]) -> Command { .envs(PROXY_ENVS.clone()) .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) - .env("GIT_PATH", GIT_PATH.as_str()); + .env("GIT_PATH", GIT_PATH.as_str()) + // dbt reports anonymous usage to dbt Labs from every invocation unless told + // not to, and a project's `flags:` block cannot override the variable. Set + // before the project's environment so a descriptor can still opt back in. + .env("DBT_SEND_ANONYMOUS_USAGE_STATS", "false"); // Both environments belong to the child. Under a sandbox they reach it through // the jail profile instead: set here, they would reach the dynamic loader that // execs nsjail itself, so an `LD_PRELOAD` from the project would run as the From 30ffdbecc15270562ceed3030c50a1cf81b1195c Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 11 Sep 2026 15:09:45 +0200 Subject: [PATCH 084/144] fix: unpin only the specifiers in the bundle a bun modules run executes (#11083) * fix: keep version pins from imported scripts in bun lockfiles Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01WCq4nkUjiZBnPPMuGYCo6w * fix: strip version pins from the bundle a bun modules run executes Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01WCq4nkUjiZBnPPMuGYCo6w * fix: unpin only module specifiers, not matching text elsewhere in the script Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01WCq4nkUjiZBnPPMuGYCo6w * docs: name the raw endpoint lock generation fetches imports through Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01WCq4nkUjiZBnPPMuGYCo6w * fix: leave require calls alone and skip spans not on a quote pair when unpinning Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01WCq4nkUjiZBnPPMuGYCo6w * test: create the bun bundle cache dir a dependency job saves into Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01WCq4nkUjiZBnPPMuGYCo6w * test: drop the lock test #11082's module test already covers Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01WCq4nkUjiZBnPPMuGYCo6w * fix: narrow the change to a fail-open strip of the modules-run bundle Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01WCq4nkUjiZBnPPMuGYCo6w * fix: unpin only the specifiers in the modules-run bundle, and log a parse fallback Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01WCq4nkUjiZBnPPMuGYCo6w --------- Co-authored-by: Claude Opus 5 (1M context) --- backend/parsers/windmill-parser-ts/src/lib.rs | 101 ++++++++++++++++++ .../parsers/windmill-parser-ts/tests/tests.rs | 41 +++++++ backend/tests/bun_jobs.rs | 51 +++++++++ backend/windmill-worker/loader.bun.js | 2 + backend/windmill-worker/src/bun_executor.rs | 15 ++- .../sync_push_auto_metadata_repro.test.ts | 2 +- 6 files changed, 209 insertions(+), 3 deletions(-) diff --git a/backend/parsers/windmill-parser-ts/src/lib.rs b/backend/parsers/windmill-parser-ts/src/lib.rs index b0ed12fb1a..271ad4835b 100644 --- a/backend/parsers/windmill-parser-ts/src/lib.rs +++ b/backend/parsers/windmill-parser-ts/src/lib.rs @@ -694,6 +694,107 @@ pub fn remove_pinned_imports(code: &str) -> anyhow::Result { Ok(content) } +/// Spans of the string literals naming a loaded module: `import`/`export … from` sources and the +/// argument of a dynamic `import()`. A `require()` call is left out: `require` is an ordinary +/// binding a script can shadow, so its argument is not known to be a module. +struct ImportSpecifierSpans(Vec); + +impl Visit for ImportSpecifierSpans { + noop_visit_type!(); + + fn visit_import_decl(&mut self, n: &swc_ecma_ast::ImportDecl) { + self.0.push(n.src.span); + } + + fn visit_export_all(&mut self, n: &swc_ecma_ast::ExportAll) { + self.0.push(n.src.span); + } + + fn visit_named_export(&mut self, n: &swc_ecma_ast::NamedExport) { + if let Some(src) = &n.src { + self.0.push(src.span); + } + } + + fn visit_call_expr(&mut self, n: &swc_ecma_ast::CallExpr) { + if let (swc_ecma_ast::Callee::Import(_), Some(arg)) = (&n.callee, n.args.first()) { + if let (None, Expr::Lit(Lit::Str(s))) = (arg.spread, &*arg.expr) { + self.0.push(s.span); + } + } + n.visit_children_with(self); + } +} + +/// Drops the `@version` from each pinned module specifier (`pkg@1.2.3/sub` -> `pkg/sub`), +/// rewriting only the specifier literals. Unlike [`remove_pinned_imports`], the same text +/// elsewhere, such as a string the script returns, stays as written. +pub fn remove_pinned_import_specifiers(code: &str) -> anyhow::Result { + let cm: Lrc = Default::default(); + let fm = cm.new_source_file( + FileName::Custom("main.d.ts".into()).into(), + code.to_string(), + ); + let mut tss = TsSyntax::default(); + tss.tsx = true; + tss.no_early_errors = true; + let lexer = Lexer::new( + Syntax::Typescript(tss), + Default::default(), + StringInput::from(&*fm), + None, + ); + let module = Parser::new_from(lexer).parse_module().map_err(|e| { + anyhow::anyhow!("Error while parsing code, it is invalid TypeScript: {e:?}") + })?; + let mut specifiers = ImportSpecifierSpans(vec![]); + specifiers.visit_module(&module); + specifiers.0.sort_by_key(|s| s.lo); + + // Spans index the parsed source, which the source map stripped of any UTF-8 BOM. + let bom = if code.starts_with('\u{feff}') { + '\u{feff}'.len_utf8() + } else { + 0 + }; + let offset = + |pos: swc_common::BytePos| pos.0.checked_sub(fm.start_pos.0).map(|o| bom + o as usize); + let mut content = String::with_capacity(code.len()); + let mut copied = 0; + for span in specifiers.0 { + // A span covers the literal's quotes. One that does not land on a matching pair is left + // as written rather than risk rewriting the wrong bytes. + let (Some(open), Some(close)) = ( + offset(span.lo), + offset(span.hi).and_then(|e| e.checked_sub(1)), + ) else { + continue; + }; + let quote = code.as_bytes().get(open); + if open >= close + || open < copied + || !matches!(quote, Some(b'"' | b'\'')) + || code.as_bytes().get(close) != quote + { + continue; + } + let Some(specifier) = code.get(open + 1..close) else { + continue; + }; + let unpinned = IMPORTS_VERSION.captures(specifier).and_then(|x| { + x.get(1) + .map(|y| format!("{}{}", y.as_str(), x.get(2).map_or("", |z| z.as_str()))) + }); + if let Some(unpinned) = unpinned.filter(|u| u != specifier) { + content.push_str(&code[copied..open + 1]); + content.push_str(&unpinned); + copied = close; + } + } + content.push_str(&code[copied..]); + Ok(content) +} + fn resolve_type_ref(type_resolver: &HashMap, typ: &mut Typ) { let mut visited = std::collections::HashSet::new(); resolve_type_ref_with_visited(type_resolver, typ, &mut visited); diff --git a/backend/parsers/windmill-parser-ts/tests/tests.rs b/backend/parsers/windmill-parser-ts/tests/tests.rs index 4309018fb4..4206dd68f8 100644 --- a/backend/parsers/windmill-parser-ts/tests/tests.rs +++ b/backend/parsers/windmill-parser-ts/tests/tests.rs @@ -4,6 +4,7 @@ mod tests { use windmill_parser::{Arg, MainArgSignature, ObjectProperty, ObjectType, Typ}; use windmill_parser_ts::{ parse_deno_signature, parse_expr_for_imports, parse_relative_imports, + remove_pinned_import_specifiers, }; #[test] @@ -33,6 +34,46 @@ mod tests { ); } + #[test] + fn test_remove_pinned_import_specifiers_rewrites_only_specifiers() { + let code = r#"// héllo +import a from "pkg@1.2.3"; +import b from "@scope/pkg@^2/sub"; +export * from "other@3"; +import rel from "./helper"; +const c = await import("dyn@4"); +const require = (v: string) => v; +const d = require("req@5"); +// pkg@1.2.3 +export const label = "pkg@1.2.3"; +"#; + assert_eq!( + remove_pinned_import_specifiers(code).unwrap(), + r#"// héllo +import a from "pkg"; +import b from "@scope/pkg/sub"; +export * from "other"; +import rel from "./helper"; +const c = await import("dyn"); +const require = (v: string) => v; +const d = require("req@5"); +// pkg@1.2.3 +export const label = "pkg@1.2.3"; +"# + ); + assert_eq!( + remove_pinned_import_specifiers("\u{feff}import a from 'pkg@1';").unwrap(), + "\u{feff}import a from 'pkg';" + ); + assert_eq!( + remove_pinned_import_specifiers( + "// a\r\n// b\r\nimport a from \"pkg@1\";\r\nimport b from 'x@2';" + ) + .unwrap(), + "// a\r\n// b\r\nimport a from \"pkg\";\r\nimport b from 'x';" + ); + } + #[test] fn test_parse_empty_main_signature() { let code = r#" diff --git a/backend/tests/bun_jobs.rs b/backend/tests/bun_jobs.rs index fc766253bc..95b4a62141 100644 --- a/backend/tests/bun_jobs.rs +++ b/backend/tests/bun_jobs.rs @@ -939,6 +939,57 @@ export function main() { return midValue(); }"#, Ok(()) } +/// A run with local modules and no lock executes the bundle its lock generation built, which +/// kept the imported script's pin; the run must still load the one copy in node_modules, and +/// leave the script's own data alone even where it matches the pinned specifier. +#[sqlx::test(fixtures("base"))] +async fn test_bun_modules_run_loads_imported_pin_from_node_modules( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + insert_deployed_bun_script( + &db, + "f/pinned_import_modules/module", + 41240002, + r#"import * as isNumber from "is-number@6.0.0"; +export const ns = isNumber; +export const label = "is-number@6.0.0";"#, + ) + .await; + + let job = JobPayload::Code(RawCode { + content: r#"import * as isNumber from "is-number"; +import { ns, label } from "/f/pinned_import_modules/module"; +import { local } from "./helper"; +export function main() { return [ns === isNumber, label, local()]; }"# + .into(), + path: Some("f/pinned_import_modules/main".into()), + language: ScriptLang::Bun, + modules: Some(std::collections::HashMap::from([( + "helper.ts".to_string(), + windmill_common::scripts::ScriptModule { + content: "export const local = () => 'local';".into(), + language: ScriptLang::Bun, + lock: None, + }, + )])), + ..RawCode::default() + }); + + let result = run_job_in_new_worker_until_complete(&db, false, job, port) + .await + .json_result() + .unwrap(); + assert_eq!( + result, + serde_json::json!([true, "is-number@6.0.0", "local"]) + ); + Ok(()) +} + #[sqlx::test(fixtures("base", "bun_edge_cases"))] async fn test_bun_shared_imports_both_styles(db: Pool) -> anyhow::Result<()> { initialize_tracing().await; diff --git a/backend/windmill-worker/loader.bun.js b/backend/windmill-worker/loader.bun.js index 9e0e071cca..7a38369dd7 100644 --- a/backend/windmill-worker/loader.bun.js +++ b/backend/windmill-worker/loader.bun.js @@ -105,6 +105,8 @@ const p = { const normalized = (isRelative ? join(dirname(file_path), pathNoExt) : pathNoExt.slice(1)).replace(/\\/g, "/"); const hash = TEMP_SCRIPT_REFS?.[normalized]; + // Lock generation substitutes `raw`: the dependency scan reads versions from the + // `pkg@version` specifiers in imported scripts, which `raw_unpinned` strips. const url = (isRelative ? `${base_internal_url}/api/w/${w_id}/scripts/RAW_GET_ENDPOINT/p/${file_path}/../${args.path}${endExt}` : `${base_internal_url}/api/w/${w_id}/scripts/RAW_GET_ENDPOINT/p/${args.path}${endExt}` diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index 4b62dac902..16f480815e 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -13,7 +13,7 @@ use itertools::Itertools; use serde_json::value::RawValue; use uuid::Uuid; -use windmill_parser_ts::remove_pinned_imports; +use windmill_parser_ts::{remove_pinned_import_specifiers, remove_pinned_imports}; use windmill_queue::{append_logs, CanceledBy, MiniPulledJob, PrecomputedAgentInfo}; @@ -1784,8 +1784,19 @@ pub async fn handle_bun_job( if modules.as_ref().is_some_and(|m| !m.is_empty()) { let bundle_path = std::path::Path::new(job_dir).join("out").join("main.js"); if bundle_path.exists() { + // The lock-generation build kept every `pkg@version` specifier, and bun resolves + // a pinned specifier outside node_modules, loading a second copy of the package. + // The bundle holds the user's code too, so only the specifiers are rewritten, and + // a bundle the parser rejects still runs as built, pins and all. let bundled = std::fs::read_to_string(&bundle_path)?; - write_file(job_dir, "main.ts", &bundled)?; + let unpinned = remove_pinned_import_specifiers(&bundled).unwrap_or_else(|e| { + tracing::warn!( + job_id = %job.id, + "could not unpin the modules bundle, running it as built: {e:#}" + ); + bundled + }); + write_file(job_dir, "main.ts", &unpinned)?; } } "\n\n--- BUN CODE EXECUTION ---\n".to_string() diff --git a/cli/test/sync_push_auto_metadata_repro.test.ts b/cli/test/sync_push_auto_metadata_repro.test.ts index e9ea167ffc..4854198d64 100644 --- a/cli/test/sync_push_auto_metadata_repro.test.ts +++ b/cli/test/sync_push_auto_metadata_repro.test.ts @@ -168,7 +168,7 @@ test( // Customer scenario: a barrel file (f/lib/errors/index.ts) re-exports from // siblings (./types.ts, ./WorkflowError.ts, ...). An importer in a different // folder imports from the barrel. On a fresh DB, the dep job for the importer -// fetches index.ts via raw_unpinned + temp_script_hash, but bun's resolver +// fetches index.ts via raw + temp_script_hash, but bun's resolver // then has to resolve the barrel's *sibling* imports — and those need to be // in TEMP_SCRIPT_REFS too. test( From 2939c2dd4b129640d87ef45c7a24c6f215a3239a Mon Sep 17 00:00:00 2001 From: AlexRV12 <71396855+AlexRV12@users.noreply.github.com> Date: Fri, 11 Sep 2026 15:12:31 +0200 Subject: [PATCH 085/144] feat: background and wait_seconds for run_script, skip preprocessor (#11092) Claude-Session: https://claude.ai/code/session_01YS9n9Mq5CER6bApudfKMdi Co-authored-by: Claude Opus 5 (1M context) --- .../copilot/chat/global/core.test.ts | 31 +++++++++++++++++-- .../components/copilot/chat/global/core.ts | 16 ++++++++-- 2 files changed, 42 insertions(+), 5 deletions(-) diff --git a/frontend/src/lib/components/copilot/chat/global/core.test.ts b/frontend/src/lib/components/copilot/chat/global/core.test.ts index cde58d8270..d8abb38cdf 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.test.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.test.ts @@ -4482,8 +4482,10 @@ describe('global AI tools', () => { const deployedForm = statuses.find((s) => s.runForm)?.runForm expect(deployedForm.submitted).toBe(true) expect(deployedForm.schema).toBeUndefined() + // skipPreprocessor: the form fills the main input schema, so a preprocessor would take + // these arguments for a webhook body and run main on its output instead. expect(JobService.runScriptByPath).toHaveBeenCalledWith( - expect.objectContaining({ requestBody: { name: 'Ada' } }) + expect.objectContaining({ requestBody: { name: 'Ada' }, skipPreprocessor: true }) ) }) @@ -4669,6 +4671,27 @@ describe('global AI tools', () => { expect(JSON.stringify(statuses)).not.toContain(huge) }) + // A dropped pass-through fails silently: the argument is accepted and ignored, and the + // run just waits out the default budget. + it('run_script detaches on background and on wait_seconds', async () => { + vi.mocked(ScriptService.getScriptByPath).mockResolvedValue({ + path: 'f/scripts/slow', + schema: { properties: { name: { type: 'string' } } } + } as any) + + // wait_seconds 0 detaches the same way, so one run of each pins both lines. + for (const detachArg of [{ background: true }, { wait_seconds: 0 }]) { + const onJobDetached = vi.fn() + await callGlobalTool( + 'run_script', + { path: 'f/scripts/slow', args: { name: 'Ada' }, ...detachArg }, + { ...toolCallbacks, onJobStarted: vi.fn(), onJobDetached } + ) + + expect(onJobDetached).toHaveBeenCalledWith('job-script-by-path') + } + }) + // A schema with no fields still opens a form: an empty one is still the Run button, and // that button is the whole confirmation this tool has. Skipping it because there is // nothing to fill in starts the script with no confirmation at all. @@ -5274,7 +5297,8 @@ describe('global AI tools', () => { locked: 'fixed', doc: bytes, token: '$var:u/ada/prod_api_key' - } + }, + skipPreprocessor: true }) expect(result).toContain('does not declare force_delete') expect(result).toContain('') @@ -5363,7 +5387,8 @@ describe('global AI tools', () => { expect(JobService.runScriptByPath).toHaveBeenCalledWith({ workspace: WORKSPACE, path: 'f/scripts/greet', - requestBody: { name: 'Grace' } + requestBody: { name: 'Grace' }, + skipPreprocessor: true }) // The model must not assume its proposal is what ran. expect(result).toContain('Ran with arguments: {"name":"Grace"}') diff --git a/frontend/src/lib/components/copilot/chat/global/core.ts b/frontend/src/lib/components/copilot/chat/global/core.ts index e5a326985c..adb4f4e6e6 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.ts @@ -909,7 +909,9 @@ const testRunScriptToolDef = createToolDef( const runScriptSchema = z.object({ path: z.string().describe('Workspace path of the deployed script to run.'), - args: testRunArgsSchema + args: testRunArgsSchema, + background: backgroundArgSchema, + wait_seconds: waitSecondsArgSchema }) const runScriptToolDef = createToolDef( @@ -5772,8 +5774,18 @@ async function runDeployedScript( // Bypassable like a test run: the posture is the user's standing answer, and a form // it parks on is a card nobody is watching. autoAcceptable: true, + background: args.background, + detachAfterMs: waitSecondsToDetachMs(args.wait_seconds), startJob: (submitted) => - JobService.runScriptByPath({ workspace, path: args.path, requestBody: submitted }) + JobService.runScriptByPath({ + workspace, + path: args.path, + requestBody: submitted, + // As the script's own run page does: the form fills the main input schema, and a + // preprocessor would take these arguments for a webhook body and hand the script + // its own output instead. + skipPreprocessor: true + }) }, ctx ) From e0a34a86a41b7ab1297cb28469d52e875570b472 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Fri, 11 Sep 2026 15:12:51 +0200 Subject: [PATCH 086/144] keep each dev server's session when worktrees share a host (#11088) * fix: keep each dev server's session when worktrees share a host Co-Authored-By: Claude Opus 5 (1M context) * fix: keep dev auth cookies host-only on non-localhost hosts Co-Authored-By: Claude Opus 5 (1M context) * fix: keep the shared dev auth cookie with ISOLATE_DEV_AUTH=0 Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Opus 5 (1M context) --- frontend/vite.config.js | 39 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/frontend/vite.config.js b/frontend/vite.config.js index 6cda86a381..a95ada7880 100644 --- a/frontend/vite.config.js +++ b/frontend/vite.config.js @@ -125,7 +125,37 @@ const remoteUrl = ? `http://localhost:${process.env.BACKEND_PORT}` : 'https://app.windmill.dev/') -const cookieDomain = process.env.ISOLATE_DEV_AUTH === '1' ? '' : 'localhost' +// Browsers scope cookies by host, not port, so dev servers sharing a host (one per +// worktree) would share one `token` and each login would sign the others out. The session +// is stored under a name tied to its backend and forwarded as `token`, the name the backend +// reads. ISOLATE_DEV_AUTH=0 keeps the shared `token` for tools reading it across instances. +const isolateDevAuth = process.env.ISOLATE_DEV_AUTH !== '0' +const authCookie = `token_${new URL(remoteUrl).host.replace(/\W/g, '_')}` + +// Isolated cookies are host-only: `Domain=localhost` is rejected when the dev server is +// reached as 127.0.0.1 or over the network. +const cookieDomain = isolateDevAuth ? '' : 'localhost' + +function isolateAuthCookie(proxy) { + if (!isolateDevAuth) return + proxy.on('proxyReq', (proxyReq, req) => { + const kept = [] + let session + for (const cookie of (req.headers.cookie ?? '').split(/;\s*/)) { + if (cookie.startsWith(`${authCookie}=`)) session = cookie.slice(authCookie.length + 1) + else if (cookie && !cookie.startsWith('token=')) kept.push(cookie) + } + if (session !== undefined) kept.push(`token=${session}`) + if (kept.length) proxyReq.setHeader('cookie', kept.join('; ')) + else proxyReq.removeHeader('cookie') + }) + proxy.on('proxyRes', (proxyRes) => { + const setCookie = proxyRes.headers['set-cookie'] + if (setCookie) { + proxyRes.headers['set-cookie'] = setCookie.map((c) => c.replace(/^token=/, `${authCookie}=`)) + } + }) +} // Cross-origin isolation headers, scoped to mirror the production predicate — // see `needs_cross_origin_isolation` in backend/windmill-api/src/static_assets.rs @@ -179,13 +209,15 @@ const config = { '^/\\.well-known/.*': { target: remoteUrl, changeOrigin: true, - cookieDomainRewrite: cookieDomain + cookieDomainRewrite: cookieDomain, + configure: isolateAuthCookie }, '^/api/w/[^/]+/s3_proxy/.*': { target: remoteUrl, changeOrigin: false, // Important for signature to be correct cookieDomainRewrite: cookieDomain, configure: (proxy, options) => { + isolateAuthCookie(proxy) proxy.on('proxyReq', (proxyReq, req, res) => { // Prevent collapsing slashes during URL normalization const originalPath = req.url @@ -197,7 +229,8 @@ const config = { '^/api/.*': { target: remoteUrl, changeOrigin: true, - cookieDomainRewrite: cookieDomain + cookieDomainRewrite: cookieDomain, + configure: isolateAuthCookie }, '^/ws/.*': { target: process.env.REMOTE_LSP ?? process.env.REMOTE_EXTRA ?? 'https://app.windmill.dev', From e7c6f85553bd8efb0f7af0f488f615ac93c496ae Mon Sep 17 00:00:00 2001 From: Guilhem Date: Fri, 11 Sep 2026 15:47:15 +0200 Subject: [PATCH 087/144] feat: give the chat the full MCP tool schema, and mark calls with the provider icon (#11086) * feat: mark MCP server lists and chat tool calls with the provider icon Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01JgzuxyafKNF2uaEeL35XQw * feat: return the full MCP tool schema from search_mcp_tools Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01JgzuxyafKNF2uaEeL35XQw * fix: bound an empty MCP search result and keep the server mark decorative Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01JgzuxyafKNF2uaEeL35XQw * fix: keep the more-matches hint and scope a marked row to its own workspace Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01JgzuxyafKNF2uaEeL35XQw --------- Co-authored-by: Claude Opus 5 (1M context) --- .../copilot/chat/AssistantMcpSection.svelte | 13 ++-- .../copilot/chat/ToolExecutionDisplay.svelte | 19 +++++ .../copilot/chat/global/mcpTools.test.ts | 62 ++++++++++++++- .../copilot/chat/global/mcpTools.ts | 76 +++++++++++-------- .../src/lib/components/copilot/chat/shared.ts | 5 ++ .../lib/components/mcp/McpServerIcon.svelte | 29 +++++++ frontend/src/lib/components/mcp/iconCache.ts | 12 +++ .../src/lib/components/mcp/mcpMenu.svelte.ts | 13 +--- .../src/lib/components/mcp/serverMark.test.ts | 45 +++++++++++ frontend/src/lib/components/mcp/serverMark.ts | 47 ++++++++++++ 10 files changed, 273 insertions(+), 48 deletions(-) create mode 100644 frontend/src/lib/components/mcp/McpServerIcon.svelte create mode 100644 frontend/src/lib/components/mcp/serverMark.test.ts create mode 100644 frontend/src/lib/components/mcp/serverMark.ts diff --git a/frontend/src/lib/components/copilot/chat/AssistantMcpSection.svelte b/frontend/src/lib/components/copilot/chat/AssistantMcpSection.svelte index 1637165f87..0ab2812b2e 100644 --- a/frontend/src/lib/components/copilot/chat/AssistantMcpSection.svelte +++ b/frontend/src/lib/components/copilot/chat/AssistantMcpSection.svelte @@ -20,6 +20,7 @@ switch that decides whether this chat carries its tools. forgetProviderKey, rememberProviderKey } from '$lib/components/mcp/iconCache' + import McpServerIcon from '$lib/components/mcp/McpServerIcon.svelte' import ConfirmationModal from '$lib/components/common/confirmationModal/ConfirmationModal.svelte' import type { Component } from 'svelte' import { ResourceService } from '$lib/gen' @@ -29,6 +30,7 @@ switch that decides whether this chat carries its tools. import { untrack } from 'svelte' import { getAiChatManager } from './aiChatManagerContext' import { clearMcpToolsCache } from './global/mcpTools' + import { forgetMcpServerMarks } from '$lib/components/mcp/serverMark' let { ws, @@ -389,8 +391,10 @@ switch that decides whether this chat carries its tools. async function refresh(target = ws) { // A path can be reconnected to a different server, so the cached tool list - // (and the readOnlyHint the confirmation gate reads) must not survive. + // (and the readOnlyHint the confirmation gate reads) must not survive — nor the + // provider mark the transcript's call rows show. clearMcpToolsCache() + forgetMcpServerMarks() await loadServers(target) // `refreshMcpServers` blanks the list when the workspace it is handed is not // the one the chat is on, so a refresh landing after a switch would take B's @@ -469,12 +473,7 @@ switch that decides whether this chat carries its tools.
    {#each servers as server (server.path)} {#snippet icon()} - {#if server.icon} - {@const Icon = server.icon} - - {:else} - - {/if} + {/snippet} {#snippet title()} {server.path} diff --git a/frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte b/frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte index f5ae22a332..620357ff87 100644 --- a/frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte +++ b/frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte @@ -40,6 +40,8 @@ import RunScriptCard from './RunScriptCard.svelte' import WebSearchSourcesDisplay from './WebSearchSourcesDisplay.svelte' import ExpandableImage from '$lib/components/common/image/ExpandableImage.svelte' + import McpServerIcon from '$lib/components/mcp/McpServerIcon.svelte' + import { resolveMcpServerMark } from '$lib/components/mcp/serverMark' interface Props { message: ToolDisplayMessage @@ -47,6 +49,11 @@ let { message }: Props = $props() + // Recorded by the call itself, from the connected-server list rather than from the + // model's arguments — which is what lets a reloaded transcript still resolve it, and + // what keeps a path the model made up from being read as a workspace resource. + const mcpServer = $derived(message.mcpServer) + const isPlanReview = $derived(message.toolName === EXIT_PLAN_MODE_TOOL) const isPlanCard = $derived(isPlanCardTool(message.toolName)) const planCopy = $derived( @@ -242,6 +249,17 @@ {/if} {/snippet} + + {#snippet serverMark()} + {#if mcpServer?.workspace && mcpServer.path} + {#await resolveMcpServerMark(mcpServer.workspace, mcpServer.path) then mark} + + {/await} + {/if} + {/snippet} + @@ -258,6 +276,7 @@ labelClass={showPreviewChip ? 'truncate' : ''} contentClass="space-y-3" headerRight={showPreviewChip ? previewChip : undefined} + headerLeft={mcpServer?.workspace ? serverMark : undefined} > {#snippet belowHeader()} diff --git a/frontend/src/lib/components/copilot/chat/global/mcpTools.test.ts b/frontend/src/lib/components/copilot/chat/global/mcpTools.test.ts index a155115d28..e5d3af8b36 100644 --- a/frontend/src/lib/components/copilot/chat/global/mcpTools.test.ts +++ b/frontend/src/lib/components/copilot/chat/global/mcpTools.test.ts @@ -228,7 +228,7 @@ describe('call results', () => { }) describe('search_mcp_tools', () => { - it('returns compact summaries without the full input schemas', async () => { + it('returns each match with its full input schema', async () => { const result = await run('search_mcp_tools', { query: 'issue' }) expect(result.matches).toEqual([ { @@ -236,7 +236,7 @@ describe('search_mcp_tools', () => { tool: 'get_issue', description: 'Get details of a GitHub issue', mode: 'read', - params: ['owner', 'repo'] + inputSchema: TOOLS[0].inputSchema } ]) }) @@ -330,6 +330,64 @@ describe('result size cap', () => { expect(callMcpToolMock).not.toHaveBeenCalled() }) + it('drops the lowest-ranked schemas before dropping the matches themselves', async () => { + getMcpToolsMock.mockResolvedValue( + ['issue_a', 'issue_b', 'issue_c'].map((name) => ({ + name, + description: 'An issue tool', + inputSchema: { + type: 'object', + properties: { body: { type: 'string', description: 'x'.repeat(9_000) } } + }, + annotations: { readOnlyHint: true } + })) + ) + const result = await run('search_mcp_tools', { query: 'issue' }) + + expect(result.matches.map((m: any) => m.tool)).toEqual(['issue_a', 'issue_b', 'issue_c']) + expect(result.matches[0].inputSchema).toBeDefined() + expect(result.matches[2].inputSchema).toBeUndefined() + expect(result.truncated).toBe(true) + }) + + // Truncating must not swallow the count of matches the score cut off: those + // tools are not in the result at all, so the model has to know to ask again. + it('keeps the more-matches hint when truncation strips schemas', async () => { + getMcpToolsMock.mockResolvedValue( + Array.from({ length: 12 }, (_, i) => ({ + name: `issue_${i}`, + description: 'An issue tool', + inputSchema: { + type: 'object', + properties: { body: { type: 'string', description: 'x'.repeat(3_000) } } + }, + annotations: { readOnlyHint: true } + })) + ) + const result = await run('search_mcp_tools', { query: 'issue' }) + + expect(result.matches).toHaveLength(10) + expect(result.note).toContain('2 more match(es)') + expect(result.note).toContain('inputSchema') + }) + + it('holds the cap when every server failed and nothing matched', async () => { + getMcpToolsMock.mockRejectedValue(new Error('x'.repeat(500))) + const servers = Array.from({ length: 50 }, (_, i) => ({ path: `u/hugo/mcp_${i}` })) + const raw = await createMcpTools(servers) + .find((t) => t.def.function.name === 'search_mcp_tools')! + .fn({ + args: { query: 'issue' }, + workspace: 'test-ws', + helpers: {}, + toolCallbacks: createToolCallbacks(), + toolId: 'tool-1' + }) + + expect(raw.length).toBeLessThanOrEqual(20_000) + expect(JSON.parse(raw).unavailableCount).toBe(50) + }) + it('truncates an oversized tools/list failure in search', async () => { getMcpToolsMock.mockRejectedValue(new Error('x'.repeat(80_000))) const result = await run('search_mcp_tools', { query: 'issue' }) diff --git a/frontend/src/lib/components/copilot/chat/global/mcpTools.ts b/frontend/src/lib/components/copilot/chat/global/mcpTools.ts index d5f16d5c54..6fd59e4a07 100644 --- a/frontend/src/lib/components/copilot/chat/global/mcpTools.ts +++ b/frontend/src/lib/components/copilot/chat/global/mcpTools.ts @@ -8,8 +8,7 @@ import { enabledMcpPaths } from '$lib/components/mcp/enabledServers' * as three static tools — search, read call, write call — instead of one * registered tool per remote tool. A server like GitHub's exposes ~90 tools, * whose schemas would otherwise be re-sent on every chat iteration; here only - * matched summaries enter the model's context, and a full input schema only - * after a call fails. + * the tools a search matched enter the model's context, with their schemas. */ type McpToolDef = GetMcpToolsResponse[number] @@ -17,7 +16,7 @@ type McpToolDef = GetMcpToolsResponse[number] export type McpServer = { path: string; editedAt?: string } const MAX_SEARCH_RESULTS = 10 -const MAX_DESCRIPTION_CHARS = 200 +const MAX_DESCRIPTION_CHARS = 1_000 const MAX_RESULT_CHARS = 20_000 // A server writes its own error text, and every enabled server can contribute // one, so search results are capped the same way call results are. @@ -139,24 +138,17 @@ function isReadOnly(tool: McpToolDef): boolean { return tool.annotations?.readOnlyHint === true } -function schemaPropertyNames(schema: unknown): string[] { - const properties = (schema as { properties?: Record } | null | undefined) - ?.properties - return properties ? Object.keys(properties) : [] -} - function truncate(text: string, max: number): string { return text.length > max ? text.slice(0, max) + '…' : text } function summarizeTool(server: McpServer, tool: McpToolDef) { - const params = schemaPropertyNames(tool.inputSchema) return { server: server.path, tool: tool.name, description: truncate(tool.description ?? '', MAX_DESCRIPTION_CHARS), mode: isReadOnly(tool) ? 'read' : 'write', - ...(params.length > 0 ? { params } : {}) + inputSchema: tool.inputSchema } } @@ -285,28 +277,44 @@ function bounded(payload: { } /** - * The search payload under the same ceiling as a call result. Server error text - * goes first: the matches are what the model asked for. + * The search payload under the same ceiling as a call result. Schemas go before + * matches: a tool listed without its schema is still callable, since a rejected + * call returns the schema, while a tool dropped from the list is not. */ function boundedSearch(payload: { matches: unknown[]; [k: string]: unknown }): string { - const { unavailable, ...rest } = payload - const dropped = Array.isArray(unavailable) ? { unavailableCount: unavailable.length } : {} - let out = JSON.stringify(payload, null, 2) + let out = JSON.stringify(payload) if (out.length <= MAX_RESULT_CHARS) return out - const matches = [...payload.matches] - const build = () => - JSON.stringify( - { - ...rest, - ...dropped, - matches, - truncated: true, - note: `Truncated to ${MAX_RESULT_CHARS} characters. Refine the query.` - }, - null, - 2 - ) + + // The caller's note says how many matches the score cut off, which truncating + // only adds to. + const { unavailable, note: scoreNote, ...rest } = payload + const dropped = Array.isArray(unavailable) ? { unavailableCount: unavailable.length } : {} + const found = payload.matches.length + const matches = payload.matches.map((m) => ({ ...(m as object) })) as Record[] + const build = () => { + const schemaless = matches.filter((m) => !m.inputSchema).length + const note = typeof scoreNote === 'string' ? [scoreNote] : [] + note.push(`Truncated to ${MAX_RESULT_CHARS} characters.`) + if (schemaless > 0) { + note.push( + `${schemaless} lower-ranked match(es) are listed without their inputSchema; calling one returns the schema if the arguments are wrong.` + ) + } + if (matches.length < found) { + note.push(`${found - matches.length} match(es) dropped — refine the query to see them.`) + } + return JSON.stringify({ ...rest, ...dropped, matches, truncated: true, note: note.join(' ') }) + } + // Dropping the server error text is the first reduction, and the only one left + // when nothing matched. out = build() + if (out.length <= MAX_RESULT_CHARS) return out + // `matches` is ordered by score, so the tail is what the query matched least. + for (let i = matches.length - 1; i >= 0; i--) { + delete matches[i].inputSchema + out = build() + if (out.length <= MAX_RESULT_CHARS) return out + } while (out.length > MAX_RESULT_CHARS && matches.length > 0) { matches.pop() out = build() @@ -353,6 +361,14 @@ function createCallTool(servers: McpServer[], mode: 'read' | 'write'): Tool<{}> }), fn: async ({ args, workspace, toolId, toolCallbacks }) => { const parsed = callMcpToolSchema.parse(args) + // Marks the row with the provider it reaches. Recorded before the listing is + // awaited, so an unreachable server still marks its own failure row, and taken + // from the connected list rather than from `parsed.server`, which is a path the + // model could point anywhere. + const named = servers.find((s) => s.path === parsed.server) + if (named) { + toolCallbacks.setToolStatus(toolId, { mcpServer: { workspace, path: named.path } }) + } // Listing is a live call to a third party: a server that has gone away // must fail this tool, not the chat loop around it. let resolved: Awaited> @@ -405,7 +421,7 @@ export function createMcpTools(servers: McpServer[]): Tool<{}>[] { def: createToolDef( searchMcpToolsSchema, 'search_mcp_tools', - 'Search the tools exposed by the MCP servers connected to this workspace (listed in the system prompt). Returns server + tool names to pass to call_mcp_read_tool or call_mcp_write_tool.' + 'Search the tools exposed by the MCP servers connected to this workspace (listed in the system prompt). Returns server + tool names to pass to call_mcp_read_tool or call_mcp_write_tool, each with the input schema its arguments must follow.' ), fn: async ({ args, workspace, toolId, toolCallbacks }) => { const parsed = searchMcpToolsSchema.parse(args) diff --git a/frontend/src/lib/components/copilot/chat/shared.ts b/frontend/src/lib/components/copilot/chat/shared.ts index adb0f63be4..bc00e415f6 100644 --- a/frontend/src/lib/components/copilot/chat/shared.ts +++ b/frontend/src/lib/components/copilot/chat/shared.ts @@ -629,6 +629,11 @@ export type ToolDisplayMessage = { autoCollapseDetails?: boolean isStreamingArguments?: boolean toolName?: string + /** What marks this row with its provider. Recorded rather than looked up, because the + * server listing lives only in memory and a reloaded transcript does not have it. The + * workspace rides along: a chat is readable from any workspace, and the same path names + * a different server in each. */ + mcpServer?: { workspace: string; path: string } showFade?: boolean actions?: ToolDisplayAction[] userQuestion?: UserQuestionDisplay diff --git a/frontend/src/lib/components/mcp/McpServerIcon.svelte b/frontend/src/lib/components/mcp/McpServerIcon.svelte new file mode 100644 index 0000000000..b3d005ad7f --- /dev/null +++ b/frontend/src/lib/components/mcp/McpServerIcon.svelte @@ -0,0 +1,29 @@ + + + + diff --git a/frontend/src/lib/components/mcp/iconCache.ts b/frontend/src/lib/components/mcp/iconCache.ts index e3c0f592e2..59ba63adfb 100644 --- a/frontend/src/lib/components/mcp/iconCache.ts +++ b/frontend/src/lib/components/mcp/iconCache.ts @@ -33,6 +33,18 @@ export function cachedProviderKey( return entry.editedAt === editedAt ? entry.key : undefined } +/** + * The stored mark for a path, ignoring `editedAt`: a transcript row has none to + * match against, and nothing acts on a mark. + */ +export function cachedProviderMark( + workspace: string, + path: string +): { key: string | null } | undefined { + const entry = read()[workspace]?.[path] + return entry ? { key: entry.key } : undefined +} + export function rememberProviderKey( workspace: string, path: string, diff --git a/frontend/src/lib/components/mcp/mcpMenu.svelte.ts b/frontend/src/lib/components/mcp/mcpMenu.svelte.ts index 847c03e61a..aea4a7b114 100644 --- a/frontend/src/lib/components/mcp/mcpMenu.svelte.ts +++ b/frontend/src/lib/components/mcp/mcpMenu.svelte.ts @@ -1,4 +1,4 @@ -import { List, Plug, Plus } from 'lucide-svelte' +import { List, Plus } from 'lucide-svelte' import type { Component } from 'svelte' import { get } from 'svelte/store' import { ResourceService } from '$lib/gen' @@ -9,6 +9,7 @@ import type { AIChatManager } from '../copilot/chat/AIChatManager.svelte' import { isMcpEnabled, setMcpEnabled } from './enabledServers' import { cachedProviderKey, rememberProviderKey } from './iconCache' import { loadProviderIcon } from './providerIcon' +import McpServerIcon from './McpServerIcon.svelte' type Row = { path: string @@ -164,19 +165,13 @@ export class McpMenu { return [ ...shown.map(({ path }) => ({ displayName: path, + icon: McpServerIcon, // Getters, not snapshots: the menu stays open across a click, and it has // to read through the live list rather than the row captured here, since // a reload replaces every row object and a getter bound to the old one // would go on reporting the state it was built with. - get icon() { - // Plug where the provider is unknown, so one nameless server does not - // pull its label out of line with the rest. - return row(path)?.icon ?? Plug - }, - // Provider icons take css lengths and ignore lucide's `size`, so without - // this one of them renders at its 24px default among 14px menu icons. get iconProps() { - return row(path)?.icon ? { width: '14px', height: '14px' } : undefined + return { icon: row(path)?.icon, size: 14 } }, get toggle() { return row(path)?.enabled ?? false diff --git a/frontend/src/lib/components/mcp/serverMark.test.ts b/frontend/src/lib/components/mcp/serverMark.test.ts new file mode 100644 index 0000000000..7e34d27284 --- /dev/null +++ b/frontend/src/lib/components/mcp/serverMark.test.ts @@ -0,0 +1,45 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { getResourceMock } = vi.hoisted(() => ({ getResourceMock: vi.fn() })) + +vi.mock('$lib/gen', () => ({ ResourceService: { getResource: getResourceMock } })) +vi.mock('./iconCache', () => ({ cachedProviderMark: () => undefined })) +vi.mock('./providerIcon', () => ({ + loadProviderIcon: async (key: string | null) => (key ? `icon:${key}` : undefined), + providerKey: (url: unknown) => + typeof url === 'string' && url.includes('linear') ? 'linear' : null +})) + +import { forgetMcpServerMarks, resolveMcpServerMark } from './serverMark' + +describe('resolveMcpServerMark', () => { + beforeEach(() => { + forgetMcpServerMarks() + getResourceMock.mockReset() + }) + + it('reads a server once and shares the answer', async () => { + getResourceMock.mockResolvedValue({ value: { url: 'https://mcp.linear.app/mcp' } }) + + const marks = await Promise.all([ + resolveMcpServerMark('ws', 'u/admin/linear_mcp'), + resolveMcpServerMark('ws', 'u/admin/linear_mcp') + ]) + + expect(marks.map((m) => m.icon)).toEqual(['icon:linear', 'icon:linear']) + expect(getResourceMock).toHaveBeenCalledTimes(1) + }) + + // A failure must not be memoized: one offline blip would otherwise leave the + // server unmarked in every later row until the page reloads. + it('retries after a failed read', async () => { + getResourceMock.mockRejectedValueOnce(new Error('offline')) + expect(await resolveMcpServerMark('ws', 'u/admin/linear_mcp')).toEqual({}) + + getResourceMock.mockResolvedValue({ value: { url: 'https://mcp.linear.app/mcp' } }) + const mark = await resolveMcpServerMark('ws', 'u/admin/linear_mcp') + + expect(mark.icon).toBe('icon:linear') + expect(getResourceMock).toHaveBeenCalledTimes(2) + }) +}) diff --git a/frontend/src/lib/components/mcp/serverMark.ts b/frontend/src/lib/components/mcp/serverMark.ts new file mode 100644 index 0000000000..c728abdead --- /dev/null +++ b/frontend/src/lib/components/mcp/serverMark.ts @@ -0,0 +1,47 @@ +import type { Component } from 'svelte' +import { ResourceService } from '$lib/gen' +import { cachedProviderMark } from './iconCache' +import { loadProviderIcon, providerKey } from './providerIcon' + +/** Windmill's own icon for a connected server's integration, when it ships one. */ +export type McpServerMark = { icon?: Component } + +// One resolution per server per session, shared by every transcript row naming it — +// a chat can hold dozens of calls against the same server. +const marks = new Map>() + +export function resolveMcpServerMark(workspace: string, path: string): Promise { + const key = `${workspace}:${path}` + let pending = marks.get(key) + if (!pending) { + // A failed read is dropped rather than memoized: one offline blip or a 401 during + // a token refresh would otherwise leave the server unmarked in every later row + // until the page reloads. + pending = load(workspace, path).catch(() => { + marks.delete(key) + return {} + }) + marks.set(key, pending) + } + return pending +} + +/** + * Forget what was resolved, for the settings section to call when it reloads the + * connections: a path can be reconnected to a different provider, and a mark held for + * the life of the page would go on marking new call rows with the old provider's icon. + */ +export function forgetMcpServerMarks() { + marks.clear() +} + +async function load(workspace: string, path: string): Promise { + const cached = cachedProviderMark(workspace, path) + if (cached) return { icon: await loadProviderIcon(cached.key) } + // Deliberately not written back to the shared cache: that entry is keyed by + // `editedAt` for the server list's sake, and storing one from here — where the + // row is a past call and `editedAt` is unknown — would make every list re-read. + const resource = await ResourceService.getResource({ workspace, path }) + const url = (resource.value as { url?: unknown } | undefined)?.url + return { icon: await loadProviderIcon(providerKey(url)) } +} From b50de8947908f1a5a4e9472afe6c0ecd25892e93 Mon Sep 17 00:00:00 2001 From: AlexRV12 <71396855+AlexRV12@users.noreply.github.com> Date: Fri, 11 Sep 2026 17:37:53 +0200 Subject: [PATCH 088/144] feat: run a deployed flow through the chat's argument form (#11085) * feat: run a deployed flow through the chat's argument form Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_015vHs7Jr4UDSbUe2KGjugMw * refactor: drop the unread dynselect helper from the deployed flow run form Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_015vHs7Jr4UDSbUe2KGjugMw * fix: skip the preprocessor when the chat runs a deployed flow Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_015vHs7Jr4UDSbUe2KGjugMw * test: pin run_flow steering with ai_evals cases Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_015vHs7Jr4UDSbUe2KGjugMw * refactor: inline the deployed flow schema and trim the eval draft check Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_015vHs7Jr4UDSbUe2KGjugMw * docs: correct the stale draft-validation comment on the flow test-run eval Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_015vHs7Jr4UDSbUe2KGjugMw --------- Co-authored-by: Claude Opus 5 (1M context) --- ai_evals/cases/global.yaml | 66 ++++++++++++++++++ .../global/initial/notify_customer_flow.json | 40 +++++++++++ .../chat/global/apiCatalogTools.test.ts | 46 +++++++++---- .../copilot/chat/global/apiCatalogTools.ts | 1 + .../copilot/chat/global/core.test.ts | 62 +++++++++++++++++ .../components/copilot/chat/global/core.ts | 67 ++++++++++++++++++- 6 files changed, 267 insertions(+), 15 deletions(-) create mode 100644 ai_evals/fixtures/frontend/global/initial/notify_customer_flow.json diff --git a/ai_evals/cases/global.yaml b/ai_evals/cases/global.yaml index 16d96d10b9..71693156a9 100644 --- a/ai_evals/cases/global.yaml +++ b/ai_evals/cases/global.yaml @@ -2046,6 +2046,72 @@ - passes the account "acme" - does not invent or guess the token's value +- id: global-test35-run-deployed-flow-with-form + prompt: |- + Run the deployed flow `f/evals/global/notify_customer` for me — the customer is `acme`. + initial: ai_evals/fixtures/frontend/global/initial/notify_customer_flow.json + runtime: + maxTurns: 8 + # A session chat is where the run card has a preview pane beside it; run_flow + # itself is offered in every chat. + sessionChat: true + validate: + draftCountExactly: 0 + toolExpect: + requiredToolsUsed: + - run_flow + # A draft may declare different arguments than the deployed version being run, so + # the names to prefill have to come from the deployed schema. + - read_workspace_item + forbiddenToolsUsed: + - test_run_flow + - call_api_endpoint + - write_flow + - deploy_workspace_item + # An empty form pushes the work back onto the user, so the prefill is part of + # what the tool is for. + toolCallArgs: + - tool: run_flow + field: args.customer + stringIncludesAnyOf: + - acme + # Running produces no draft, and the judge cannot observe runs; validate via tool use. + skipJudge: true + judgeChecklist: + - runs the deployed flow through run_flow rather than a preview test run or a raw API endpoint + - passes the customer "acme" so the confirmation form comes up prefilled + +- id: global-test36-draft-flow-test-run-not-deployed + prompt: |- + Update the `calculate_total` step of `f/evals/global/process_invoice` so it applies 8% tax and + returns `subtotal`, `tax` and `total`, then run it to check it works. + Keep it as an AI draft only; do not deploy or save it. + initial: ai_evals/fixtures/frontend/global/initial/process_invoice_flow.json + runtime: + maxTurns: 10 + validate: + draftCountExactly: 1 + requiredDrafts: + - type: flow + path: f/evals/global/process_invoice + toolExpect: + # A one-step flow is as well checked by running the step as the whole flow, so both + # count: what matters is that the run is against the draft. + requiredToolsAnyOf: + - [test_run_flow, test_run_step] + # The draft is what the user asked to check, and run_flow would run the deployed + # version instead — the edit would not be in what ran. + forbiddenToolsUsed: + - run_flow + - call_api_endpoint + - deploy_workspace_item + # The judge cannot observe runs, and the edit's content is already pinned by + # global-test5 on this fixture; what this case guards is where the run went. + skipJudge: true + judgeChecklist: + - creates an AI draft of f/evals/global/process_invoice applying 8% tax + - does not deploy or save the draft + - id: global-undo-created-draft prompt: |- Create a draft Postgres resource at `u/admin/scratch_db` for host db.example.com port 5432, database `orders`, user `app`, and tell me what fields it ended up with. diff --git a/ai_evals/fixtures/frontend/global/initial/notify_customer_flow.json b/ai_evals/fixtures/frontend/global/initial/notify_customer_flow.json new file mode 100644 index 0000000000..d8cbf37ace --- /dev/null +++ b/ai_evals/fixtures/frontend/global/initial/notify_customer_flow.json @@ -0,0 +1,40 @@ +{ + "workspace": { + "flows": [ + { + "path": "f/evals/global/notify_customer", + "summary": "Notify a customer", + "description": "Sends a notification to the named customer.", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "customer": { + "type": "string" + } + }, + "required": ["customer"] + }, + "value": { + "modules": [ + { + "id": "notify", + "summary": "Send the notification", + "value": { + "type": "rawscript", + "language": "bun", + "content": "export async function main(customer: string) {\n return `Notified ${customer}`\n}\n", + "input_transforms": { + "customer": { + "type": "javascript", + "expr": "flow_input.customer" + } + } + } + } + ] + } + } + ] + } +} diff --git a/frontend/src/lib/components/copilot/chat/global/apiCatalogTools.test.ts b/frontend/src/lib/components/copilot/chat/global/apiCatalogTools.test.ts index deb3694e99..0b1ea0952e 100644 --- a/frontend/src/lib/components/copilot/chat/global/apiCatalogTools.test.ts +++ b/frontend/src/lib/components/copilot/chat/global/apiCatalogTools.test.ts @@ -101,6 +101,19 @@ const CATALOG = [ }, body_schema: { type: 'object', additionalProperties: true } }, + { + name: 'cancelQueuedJob', + description: 'Cancel a queued job', + instructions: '', + path: '/w/{workspace}/jobs_u/queue/cancel/{id}', + method: 'POST', + path_params_schema: { + type: 'object', + properties: { workspace: { type: 'string' }, id: { type: 'string' } }, + required: ['workspace', 'id'] + }, + body_schema: { type: 'object', additionalProperties: true } + }, { name: 'runFlowByPath', description: 'Run flow by path', @@ -179,7 +192,7 @@ describe('call_api_get', () => { const unknown = await run('call_api_get', { name: 'nope' }) expect(unknown.error).toContain('search_api_endpoints') - const mutating = await run('call_api_get', { name: 'runFlowByPath' }) + const mutating = await run('call_api_get', { name: 'cancelQueuedJob' }) expect(mutating.error).toContain('call_api_endpoint') const deleting = await run('call_api_endpoint', { name: 'deleteSchedule' }) @@ -191,17 +204,22 @@ describe('call_api_get', () => { expect(search.matches.map((m: any) => m.name)).not.toContain('deleteScriptByHash') }) - // Left reachable, this endpoint is the way around the argument form: it runs the - // deployed script on the model's arguments, unstripped and unshown. - it('refuses a deployed script run, pointing at run_script', async () => { - const called = await run('call_api_endpoint', { name: 'runScriptByPath' }) - expect(called.error).toContain('run_script') - expect(called.success).toBe(false) + // Left reachable, these endpoints are the way around the argument form: they run the + // deployed runnable on the model's arguments, unstripped and unshown. + it('refuses a deployed run, pointing at run_script and run_flow', async () => { + for (const [name, tool, query] of [ + ['runScriptByPath', 'run_script', 'run deployed script'], + ['runFlowByPath', 'run_flow', 'run deployed flow'] + ]) { + const called = await run('call_api_endpoint', { name }) + expect(called.error).toContain(tool) + expect(called.success).toBe(false) - // And it is gone from search, so the model is redirected before it ever calls. - const search = await run('search_api_endpoints', { query: 'run deployed script' }) - expect(search.matches.map((m: any) => m.name)).not.toContain('runScriptByPath') - expect(search.covered_by_dedicated_tools?.join(' ')).toContain('run_script') + // And it is gone from search, so the model is redirected before it ever calls. + const search = await run('search_api_endpoints', { query }) + expect(search.matches.map((m: any) => m.name)).not.toContain(name) + expect(search.covered_by_dedicated_tools?.join(' ')).toContain(tool) + } }) it('refuses draft-blind item reads and lists, pointing at the draft-aware tools', async () => { @@ -259,11 +277,11 @@ describe('call_api_endpoint', () => { }) vi.stubGlobal('fetch', fetchMock) const result = await run('call_api_endpoint', { - name: 'runFlowByPath', - params: { path: 'u/me/myflow' }, + name: 'cancelQueuedJob', + params: { id: 'job/1' }, body: { args: { n: 1 } } }) - expect(fetchMock).toHaveBeenCalledWith('/api/w/test-ws/jobs/run/f/u%2Fme%2Fmyflow', { + expect(fetchMock).toHaveBeenCalledWith('/api/w/test-ws/jobs_u/queue/cancel/job%2F1', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ args: { n: 1 } }) diff --git a/frontend/src/lib/components/copilot/chat/global/apiCatalogTools.ts b/frontend/src/lib/components/copilot/chat/global/apiCatalogTools.ts index 2271ff91e0..42d45916eb 100644 --- a/frontend/src/lib/components/copilot/chat/global/apiCatalogTools.ts +++ b/frontend/src/lib/components/copilot/chat/global/apiCatalogTools.ts @@ -37,6 +37,7 @@ const COVERED_ENDPOINTS: Record = { listResource: 'list_workspace_items (it includes your drafts)', listSchedules: 'list_workspace_items (it includes your drafts)', runScriptByPath: 'run_script (it shows the user an argument form to confirm)', + runFlowByPath: 'run_flow (it shows the user an argument form to confirm)', deleteScriptByPath: 'delete_workspace_item', deleteScriptByHash: 'delete_workspace_item', deleteFlowByPath: 'delete_workspace_item', diff --git a/frontend/src/lib/components/copilot/chat/global/core.test.ts b/frontend/src/lib/components/copilot/chat/global/core.test.ts index d8abb38cdf..90ff8a8f77 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.test.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.test.ts @@ -5081,6 +5081,68 @@ describe('global AI tools', () => { }) }) + // What separates a deployed run from a test run of the same flow: the form is built from the + // deployed schema rather than the draft's, and the editor open on that path is left alone — + // it holds the draft, so a deployed run painted into its graph would show steps that are not + // the ones running. + it('run_flow forms on the deployed flow and leaves the live editor alone', async () => { + seedBackendDraft( + 'flow', + 'u/admin/deployed_and_drafted', + { + path: 'u/admin/deployed_and_drafted', + summary: 'Draft of the deployed flow', + value: { modules: [{ id: 'draft_step', value: { type: 'identity' } }] }, + schema: { type: 'object', properties: { draft_only: { type: 'string' } } }, + edited_by: '', + edited_at: '', + archived: false, + extra_perms: {} + }, + { workspace: WORKSPACE } + ) + UserDraft.setLiveEditorDraft({ + workspace: WORKSPACE, + itemKind: 'flow', + storagePath: 'u/admin/deployed_and_drafted', + effectivePath: 'u/admin/deployed_and_drafted' + }) + vi.mocked(FlowService.getFlowByPath).mockResolvedValueOnce({ + path: 'u/admin/deployed_and_drafted', + summary: 'Deployed flow', + value: { modules: [{ id: 'deployed_step', value: { type: 'identity' } }] }, + schema: FLOW_NAME_SCHEMA + } as any) + const testActiveFlow = vi.fn(async () => 'job-live-flow') + + let form: any + await withCompletedTestJob(() => + callGlobalTool( + 'run_flow', + { path: 'u/admin/deployed_and_drafted', args: { name: 'Ada' } }, + { + ...toolCallbacks, + requestRunArgs: async (_toolId, f) => { + form = f + return { name: 'Grace' } + } + }, + { testActiveFlow } + ) + ) + + expect(form.runnableKind).toBe('flow') + expect(form.schema?.properties).toEqual(FLOW_NAME_SCHEMA.properties) + expect(testActiveFlow).not.toHaveBeenCalled() + expect(JobService.runFlowPreview).not.toHaveBeenCalled() + expect(JobService.runFlowByPath).toHaveBeenCalledWith({ + workspace: WORKSPACE, + path: 'u/admin/deployed_and_drafted', + requestBody: { name: 'Grace' }, + skipPreprocessor: true + }) + }) + it('test_run_step previews rawscript steps from the draft flow', async () => { const content = 'export async function main(name: string) {\n\treturn name.toUpperCase()\n}' await callGlobalTool('write_flow', { diff --git a/frontend/src/lib/components/copilot/chat/global/core.ts b/frontend/src/lib/components/copilot/chat/global/core.ts index adb4f4e6e6..5883c5ae0d 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.ts @@ -935,6 +935,20 @@ const testRunFlowToolDef = createToolDef( { strict: false } ) +const runFlowSchema = z.object({ + path: z.string().describe('Workspace path of the deployed flow to run.'), + args: testRunArgsSchema, + background: backgroundArgSchema, + wait_seconds: waitSecondsArgSchema +}) + +const runFlowToolDef = createToolDef( + runFlowSchema, + 'run_flow', + 'Run a DEPLOYED flow for real, under the user\'s own permissions. Fill in every argument you can infer: the user gets an argument form prefilled with `args` and decides what runs. For a secret argument prefer `$var:` naming an existing workspace variable; a literal is minted into a short-lived secret before the run, but stays in this call. A required file is the user\'s to attach, so call this even when you cannot supply one rather than asking in chat. Use only when the user names the deployed version ("the deployed X", "in production", "for real"); otherwise use test_run_flow.', + { strict: false } +) + const testRunStepSchema = z.object({ path: z.string().describe('Workspace path of the flow containing the step to test.'), stepId: z.string().describe('The id of the step/module to test.'), @@ -1350,7 +1364,7 @@ ${pipelineBullet} : ' Pass items (":" entries naming the items you changed) so the review is scoped to them — omitting items preselects every pending change in the workspace' }, or mode ("draft" or "fork") to force which comparison is shown. Prefer offering this review page over calling deploy_workspace_item directly when several items changed. - For a Windmill operation no other tool covers (workers, queue state, a run's args, ...), use search_api_endpoints to find a REST endpoint, then call_api_get for reads or call_api_endpoint for mutations (the user is asked to confirm those). Always prefer a dedicated tool when one exists; endpoints for authoring or deleting scripts, flows, apps, schedules, resources, or variables are not available through the API catalog tools — use the draft tools and delete_workspace_item instead. -- Default to test_run_script, test_run_flow, or test_run_step for any run request, an existing script included; they prefer drafts and need no deployment. Use run_script only when the user names the deployed version ("the deployed X", "in production", "for real") — a bare "run X" is not that. For run_script, read the item with read_workspace_item version: "deployed" first so the arguments match the deployed schema, and fill in every one you can infer. test_run_script, run_script and test_run_flow all show the user an argument form prefilled with what you sent, so fill in every argument you can infer rather than asking for it in chat. runFlowByPath from the API catalog is the exception — it runs a deployed flow with no form at all: only for a flow the user asked to run deployed. +- Default to test_run_script, test_run_flow, or test_run_step for any run request, an existing script included; they prefer drafts and need no deployment. Use run_script or run_flow only when the user names the deployed version ("the deployed X", "in production", "for real") — a bare "run X" is not that. For those two, read the item with read_workspace_item version: "deployed" first so the arguments match the deployed schema. test_run_script, test_run_flow, run_script and run_flow all show the user an argument form prefilled with what you sent, so fill in every argument you can infer rather than asking for it in chat. - When a required decision is ambiguous, use askUserQuestion with two to ten clear proposed answer strings instead of guessing. The user can also type a custom answer when none of the proposed answers fit. Set multiSelect: true only when the answers can genuinely co-apply and the user may pick several (not mutually exclusive). - When the user asks you to remember a lasting preference, always/never do something, or change/stop a behavior going forward, call update_user_instructions to persist it. It edits only the USER INSTRUCTIONS block (not WORKSPACE INSTRUCTIONS). Keep each instruction concise; do not use it for one-off requests scoped to the current task. - Keep context targeted.${ @@ -3704,6 +3718,19 @@ export const globalTools: Tool<{}>[] = [ showDetails: true, autoCollapseDetails: false }, + { + def: runFlowToolDef, + fn: async (ctx) => { + const parsed = runFlowSchema.parse(ctx.args) + return runDeployedFlow(parsed, ctx) + }, + bypassedByAutoAccept: true, + streamingLabel: 'Preparing the run form...', + confirmationMessage: 'Run a deployed flow', + queuedLabel: (args) => `Run ${args?.path ?? 'a flow'}`, + showDetails: true, + autoCollapseDetails: false + }, { def: testRunStepToolDef, fn: async (ctx) => { @@ -5791,6 +5818,44 @@ async function runDeployedScript( ) } +async function runDeployedFlow( + args: z.infer, + ctx: WriteDraftCtx +): Promise { + const { workspace } = ctx + // No live editor is driven here as a test run drives one: that editor holds the draft, and a + // deployed run painted into its graph would show steps that are not the ones running. + const flow = await FlowService.getFlowByPath({ workspace, path: args.path }) + return runThroughForm( + { + path: args.path, + schema: (flow.schema as Record) ?? {}, + summary: flow.summary, + kind: 'run', + // No code/lang: the dynamic-option pickers come from the deployed flow, not an inline copy. + schemaNoun: 'deployed', + toolName: 'run_flow', + proposed: args.args, + startMessage: `Running "${args.path}"...`, + contextName: 'flow', + autoAcceptable: true, + background: args.background, + detachAfterMs: waitSecondsToDetachMs(args.wait_seconds), + startJob: (submitted) => + JobService.runFlowByPath({ + workspace, + path: args.path, + requestBody: submitted, + // As the flow's own run page does: the form fills the main input schema, and a + // preprocessor would take these arguments for a webhook body and hand the flow + // its own output instead. + skipPreprocessor: true + }) + }, + ctx + ) +} + async function testRunFlowByPath( args: z.infer, ctx: WriteDraftCtx From 0d767d00fb340b6684807ec2c6892c648e940e47 Mon Sep 17 00:00:00 2001 From: Diego Imbert <70353967+diegoimbert@users.noreply.github.com> Date: Fri, 11 Sep 2026 18:51:00 +0200 Subject: [PATCH 089/144] refactor: make the acting workspace and user explicit in the entity editors (#11031) * refactor: make the acting workspace and user explicit in the entity editors Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YLxwAsiXJ1Au8CBDBmH7iY * fix: resolve the acting user in new-item mode and for the navigation workspace Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YLxwAsiXJ1Au8CBDBmH7iY * fix: discard acting-user lookups that no longer describe the acting workspace Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YLxwAsiXJ1Au8CBDBmH7iY * refactor: own the acting-user resolution in one composable Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YLxwAsiXJ1Au8CBDBmH7iY * fix: key the acting-user cache by a Map and re-ask after a failed lookup Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YLxwAsiXJ1Au8CBDBmH7iY * fix: re-ask a failed acting-user lookup when an editor opens a new session Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YLxwAsiXJ1Au8CBDBmH7iY * fix: forget a failed acting-user lookup when its workspace stops being the acting one Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YLxwAsiXJ1Au8CBDBmH7iY * fix: drop a stale acting-user refusal on arrival rather than on departure Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YLxwAsiXJ1Au8CBDBmH7iY * fix: let the navigation user answer for the navigation workspace unconditionally Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YLxwAsiXJ1Au8CBDBmH7iY * docs: mark prototype-key workspace ids as unsupported by the entity editors Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YLxwAsiXJ1Au8CBDBmH7iY --------- Co-authored-by: Claude Opus 5 (1M context) --- frontend/src/lib/actingUser.svelte.ts | 74 ++++++++++ frontend/src/lib/components/Path.svelte | 34 +++-- .../src/lib/components/ResourceEditor.svelte | 128 +++++++++--------- .../components/ResourceEditorDrawer.svelte | 23 ++-- .../src/lib/components/ResourceForm.svelte | 19 ++- .../src/lib/components/VariableEditor.svelte | 54 ++++---- .../src/lib/components/VariableForm.svelte | 17 ++- .../schedules/ScheduleEditorInner.svelte | 29 +++- .../(root)/(logged)/resources/+page.svelte | 1 + .../(root)/(logged)/variables/+page.svelte | 6 +- 10 files changed, 261 insertions(+), 124 deletions(-) create mode 100644 frontend/src/lib/actingUser.svelte.ts diff --git a/frontend/src/lib/actingUser.svelte.ts b/frontend/src/lib/actingUser.svelte.ts new file mode 100644 index 0000000000..30da65ff83 --- /dev/null +++ b/frontend/src/lib/actingUser.svelte.ts @@ -0,0 +1,74 @@ +import { untrack } from 'svelte' +import { fromStore } from 'svelte/store' +import { SvelteMap } from 'svelte/reactivity' +import { userStore, workspaceStore, type UserExt } from '$lib/stores' +import { getWorkspaceRole, type RoleLookup } from '$lib/user' + +/** + * The user acting in a workspace that is not necessarily the one the top nav points at — an AI + * session or a workspace-specific variant acts on a workspace the nav deliberately is not on. + * + * `$userStore` answers for the navigation workspace at no cost, exactly as every permission check + * in the app did before this hook existed — including when it holds nobody, which reads as unknown + * and refuses. Every other workspace is looked up, and an unresolved user there is `undefined`: it + * must never fall back to the navigation user, whose rights belong to another workspace. + * `canWrite`/`isOwner` refuse for an unknown user, which is the only safe answer. A caller that + * must not render that refusal as a denial asks `resolved` first. + */ +export function useActingUser(workspace: () => string | undefined) { + const navWorkspace = fromStore(workspaceStore) + const navUser = fromStore(userStore) + const looked = new SvelteMap() + // The workspace this effect last acted on, so arriving at one is distinguishable from the + // effect re-running while already there. + let asking: string | undefined + + $effect(() => { + const ws = workspace() + if (asking !== ws) { + asking = ws + // Dropped on the way *in*, not on the way out: a lookup that fails after the acting + // workspace has already moved on has no entry to clear at the moment it is left, so + // clearing it there would keep a refusal that no attempt is behind any more. + if (ws && untrack(() => looked.get(ws)?.kind) === 'lookup_failed') looked.delete(ws) + } + if (!ws || ws === navWorkspace.current) return + // Any settled answer stops the asking, a failure included — otherwise recording one + // would re-enter this effect and loop. + if (looked.has(ws)) return + untrack(() => { + // Memoized process-wide, so two components pointed at the same workspace share one + // request rather than each issuing their own. + getWorkspaceRole(ws).then((lookup) => looked.set(ws, lookup)) + }) + }) + + function userIn(ws: string | undefined): UserExt | undefined { + if (!ws) return undefined + if (ws === navWorkspace.current) return navUser.current + const lookup = looked.get(ws) + return lookup?.kind === 'resolved' ? lookup.user : undefined + } + + return { + /** The acting user in `ws`, or `undefined` when it is not known. Only workspaces this + * hook has been pointed at are looked up; the rest read as unknown. */ + in: userIn, + /** Whether `ws` has an answer at all — a user, or a lookup that came back without one. + * The navigation workspace always has one: `$userStore`, "nobody" included. */ + resolved: (ws: string | undefined): boolean => + !!ws && (ws === navWorkspace.current || looked.has(ws)), + get current(): UserExt | undefined { + return userIn(workspace()) + }, + /** Drop the lookups that came back empty so they are asked again. Arriving at a + * workspace already does this; a long-lived editor must call this too when it starts a + * fresh session on the workspace it is already on, or a `whoami` that happened to fail + * pins it to "unknown user" for as long as it stays there. */ + forgetFailures(): void { + for (const [ws, lookup] of looked) { + if (lookup.kind === 'lookup_failed') looked.delete(ws) + } + } + } +} diff --git a/frontend/src/lib/components/Path.svelte b/frontend/src/lib/components/Path.svelte index 37eef75bf5..b45d980246 100644 --- a/frontend/src/lib/components/Path.svelte +++ b/frontend/src/lib/components/Path.svelte @@ -26,7 +26,7 @@ AzureTriggerService, EmailTriggerService } from '$lib/gen' - import { superadmin, userStore, workspaceStore } from '$lib/stores' + import { superadmin, userStore, workspaceStore, type UserExt } from '$lib/stores' import { createEventDispatcher, getContext, untrack } from 'svelte' import { writable } from 'svelte/store' import { Alert, Button } from './common' @@ -85,6 +85,11 @@ * workspace when the editor operates on a workspace other than the one the * top nav points at (see the sessions preview / dev-workspace flows). */ workspaceOverride?: string + /** The user acting in `workspaceOverride`, for the owner suggestion and the folder + * write flags. Omit it to stand in the navigation `$userStore`, who is a member of + * the navigation workspace only; pass `null` for "not known (yet)", which that user + * must not answer for either. */ + actingUser?: UserExt | null /** One path that does not count as taken, for a caller creating something that may * already have written there itself — a setup flow correcting its own failed attempt. * Every other existing path is still refused. */ @@ -110,11 +115,16 @@ size = 'md', drawerOffset = 0, workspaceOverride = undefined, + actingUser = undefined, allowedExistingPath = undefined, warnOnRename = true }: Props = $props() let ws = $derived(workspaceOverride ?? $workspaceStore) + // Sole place this component falls back to the ambient user, and only for a caller that + // passed none; everything below reads `user`, so a caller acting on another workspace is + // never mixed with the navigation user's memberships. + let user = $derived(actingUser === undefined ? $userStore : (actingUser ?? undefined)) $effect.pre(() => { if (path == undefined) { @@ -169,17 +179,17 @@ export async function reset() { if (path == '' || path == 'u//' || path?.startsWith('tmp/') || path?.startsWith('hub/')) { - if ($lastMetaUsed == undefined || $lastMetaUsed.owner != $userStore?.username) { + if ($lastMetaUsed == undefined || $lastMetaUsed.owner != user?.username) { meta = { ownerKind: hideUser ? 'folder' : 'user', name: fullNamePlaceholder ?? random_adj() + '_' + namePlaceholder, owner: '' } if (!hideUser) { - if ($userStore?.username?.includes('@')) { - meta.owner = $userStore!.username.split('@')[0].replace(/[^a-zA-Z0-9_]/g, '') + if (user?.username?.includes('@')) { + meta.owner = user!.username.split('@')[0].replace(/[^a-zA-Z0-9_]/g, '') } else { - meta.owner = $userStore!.username! + meta.owner = user!.username! } } } else { @@ -229,9 +239,9 @@ .map((x) => ({ name: x, write: - $userStore?.folders?.includes(x) == true || - ($userStore?.is_admin ?? false) || - ($userStore?.is_super_admin ?? false) + user?.folders?.includes(x) == true || + (user?.is_admin ?? false) || + (user?.is_super_admin ?? false) })) ) } @@ -423,7 +433,7 @@ }) }) $effect.pre(() => { - if (ws && $userStore) { + if (ws && user) { untrack(() => { loadFolders() initPath() @@ -506,7 +516,7 @@ } else { // 'group' is unreachable here (Select only offers user/folder) // but validateName still accepts it for forward-compat. - meta.owner = $userStore?.username?.split('@')[0] ?? '' + meta.owner = user?.username?.split('@')[0] ?? '' } } } @@ -520,7 +530,7 @@
    {#if meta.ownerKind === 'user'} {@const userOwnerDisabled = - disabled || !($superadmin || ($userStore?.is_admin ?? false)) || disableEditing} + disabled || !($superadmin || (user?.is_admin ?? false)) || disableEditing}
  • feature adoption (counts of which flow, script, trigger, worker and data table @@ -1150,8 +1150,8 @@ the flow editor, which skin approval steps are given, how data tables and their migrations are set up and used, how often an empty workspace home is seen, how often the home page’s create menu and hub-project picker are opened and from which entry - point, and the name of any public hub project imported from the home page and how far - that import got, last 30 days)
  • feature adoption (counts of which flow, script, trigger, worker and data table diff --git a/frontend/src/lib/components/Login.svelte b/frontend/src/lib/components/Login.svelte index e4b73478aa..da032717f6 100644 --- a/frontend/src/lib/components/Login.svelte +++ b/frontend/src/lib/components/Login.svelte @@ -1,4 +1,5 @@ + +
    + (accountSetup.open = true)} + /> + {#if isCollapsed} + + + + + + {/if} +
    diff --git a/frontend/src/lib/components/sidebar/FinishAccountSetup.svelte b/frontend/src/lib/components/sidebar/FinishAccountSetup.svelte new file mode 100644 index 0000000000..c565ccfad2 --- /dev/null +++ b/frontend/src/lib/components/sidebar/FinishAccountSetup.svelte @@ -0,0 +1,173 @@ + + + +
    +

    + Your account {email} was created from an invite and + has no sign-in method of its own yet. Pick one so you can come back any time. +

    + + {#if logins.length === 0 && !saml && !passwordAllowed} +

    + No sign-in method is available on this instance right now; ask an administrator. +

    + {/if} + {#if logins.length > 0 || saml} +
    + Sign in with a provider +
    + {#each logins as login (login.type)} + {@const Icon = icons[login.type]} + + {/each} + {#if saml} + + {/if} +
    +

    + Sign in to the provider as {email}; a different address is refused and you stay signed in + here. +

    +
    + {#if passwordAllowed} +
    +
    + or +
    +
    + {/if} + {/if} + + {#if passwordAllowed} +
    + Set a password +
    + + +
    +

    A password account keeps signing in with the password only.

    +
    + {/if} +
    +     diff --git a/frontend/src/lib/components/sidebar/MenuButton.svelte b/frontend/src/lib/components/sidebar/MenuButton.svelte index 71d88b071b..3d0286a4af 100644 --- a/frontend/src/lib/components/sidebar/MenuButton.svelte +++ b/frontend/src/lib/components/sidebar/MenuButton.svelte @@ -51,6 +51,9 @@ // Accessible name when the visible label is absent or only shown some of // the time, so the button stays announceable in every state. ariaLabel?: string | undefined + // Classes for the label line only — `class` reaches the button, the label and the + // sublabel alike, which is the wrong tool for colouring one line of the two. + labelClass?: string | undefined } let { @@ -75,7 +78,8 @@ showChevron = false, emphasizeLabel = false, disableTitle = false, - ariaLabel = undefined + ariaLabel = undefined, + labelClass = undefined }: Props = $props() let buttonRef: HTMLButtonElement | HTMLAnchorElement | undefined = $state(undefined) @@ -161,7 +165,8 @@ 'whitespace-pre truncate w-full', emphasizeLabel ? 'text-primary text-sm font-semibold' : sidebarClasses.text, 'transition-all', - classNames + classNames, + labelClass )} title={disableTitle ? undefined : label} > diff --git a/frontend/src/lib/components/sidebar/SettingsMenu.svelte b/frontend/src/lib/components/sidebar/SettingsMenu.svelte index b7a3a363bb..857e422bdd 100644 --- a/frontend/src/lib/components/sidebar/SettingsMenu.svelte +++ b/frontend/src/lib/components/sidebar/SettingsMenu.svelte @@ -17,7 +17,8 @@ Newspaper, Crown, Gauge, - Trash2 + Trash2, + KeyRound } from 'lucide-svelte' import { base } from '$app/paths' import { goto } from '$lib/navigation' @@ -35,6 +36,7 @@ import SideBarNotification from './SideBarNotification.svelte' import { markChangelogsOpened, readRecentChangelogs } from './changelogs' import { USER_SETTINGS_HASH, SUPERADMIN_SETTINGS_HASH } from './settings' + import { accountSetup } from './accountSetup.svelte' import { EXECUTIONS_HINT } from './executionsHint' import { userWorkspaces, @@ -207,6 +209,10 @@ : []) ]) + // An account entered through an invite link that still has no credentials of its own; + // the entry (and the sidebar banner it echoes) disappears once it does. + let pendingSetup = $derived(accountSetup.pending) + const items = $derived([ { displayName: 'Help', @@ -226,6 +232,17 @@ : ($userStore?.email ?? 'User'), icon: $userStore?.is_admin || $userStore?.non_member ? Crown : User, submenuItems: [ + ...(pendingSetup + ? [ + { + displayName: 'Finish account setup', + icon: KeyRound, + // The dropdown closes on this click; the modal opens once it is gone so its own + // buttons don't compete with the menu's outside-click handling. + action: () => setTimeout(() => (accountSetup.open = true), 50) + } + ] + : []), { displayName: 'Account settings', icon: Settings, diff --git a/frontend/src/lib/components/sidebar/SidebarUsage.svelte b/frontend/src/lib/components/sidebar/SidebarUsage.svelte index 1625ed83e2..6ecf81fa75 100644 --- a/frontend/src/lib/components/sidebar/SidebarUsage.svelte +++ b/frontend/src/lib/components/sidebar/SidebarUsage.svelte @@ -1,8 +1,18 @@ + + @@ -902,6 +911,18 @@ /> {/snippet} + +{#snippet accountSetupBanner(collapsed: boolean)} + {#if accountSetup.pending} +
    + +
    + {/if} +{/snippet} + {#snippet brandMark(collapsed: boolean)} @@ -929,6 +950,13 @@ {/snippet} +{#if accountSetup.pending} + accountSetup.refresh()} + /> +{/if} {#if page.status == 404} @@ -1101,6 +1129,7 @@ {/if}
    + {@render accountSetupBanner(false)}
    @@ -1239,6 +1268,7 @@ {/if}
    + {@render accountSetupBanner(isCollapsed)}
    diff --git a/frontend/src/routes/(root)/(logged)/user/(user)/onboarding/+page.svelte b/frontend/src/routes/(root)/(logged)/user/(user)/onboarding/+page.svelte index 34a9534117..035dbfd3b7 100644 --- a/frontend/src/routes/(root)/(logged)/user/(user)/onboarding/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/user/(user)/onboarding/+page.svelte @@ -25,6 +25,7 @@ MessageCircleCode } from 'lucide-svelte' import { sendUserToast } from '$lib/toast' + import { onboardingProfile } from '$lib/onboardingProfile' // Define step names as constants for better maintainability const STEP_SOURCE = 'source' @@ -51,6 +52,25 @@ // The survey was skipped, so the last step has nothing to go back to. let skippedSurvey = $state(false) + // An invited account arrives with the survey already answered: the invite that brought + // them here is how they heard about us, and their use case was researched before it was + // sent. Neither question is asked; the known source is recorded and they go straight to + // naming their workspace. Resolved before first paint: rendering a survey step and + // yanking it away a frame later reads as a glitch. + let invitedTouchPoint = $state(null) + let profileReady = $state(false) + async function loadInviteProfile() { + const profile = await onboardingProfile() + if (profile?.touch_point) { + invitedTouchPoint = profile.touch_point + // An account that already has somewhere to go leaves from here; painting the + // survey behind that navigation would show a step this account never takes. + if (await skip()) return + } + profileReady = true + } + loadInviteProfile() + async function loadWorkspaceStep() { try { const [workspaces, invites] = await Promise.all([ @@ -172,30 +192,36 @@ } } - async function skip() { + /** Declines the survey; true when that left onboarding altogether. */ + async function skip(): Promise { isSubmitting = true try { + // The known source still counts when the rest of the survey is declined. await UserService.submitOnboardingData({ - requestBody: {} + requestBody: invitedTouchPoint ? { touch_point: invitedTouchPoint } : {} }) } catch (error) { console.error('Error skipping onboarding:', error) - } finally { - await workspaceStepReady - isSubmitting = false - // Skipping the survey is not skipping naming the workspace: the questions are ours, - // the workspace is theirs. - skippedSurvey = true - if (alreadyPlaced) { - leaveOnboarding() - } else { - currentStep = STEP_WORKSPACE - } } + await workspaceStepReady + isSubmitting = false + // Skipping the survey is not skipping naming the workspace: the questions are ours, + // the workspace is theirs. + skippedSurvey = true + if (alreadyPlaced) { + await leaveOnboarding() + return true + } + currentStep = STEP_WORKSPACE + return false } -{#if currentStep === STEP_SOURCE} +{#if !profileReady} + + +{:else if currentStep === STEP_SOURCE}
    @@ -328,13 +354,16 @@ {/snippet} -
    -
    -
    -
    -
    + {#if !invitedTouchPoint} + +
    +
    +
    +
    +
    +
    -
    + {/if}
    {/if} diff --git a/frontend/src/routes/(root)/+layout.svelte b/frontend/src/routes/(root)/+layout.svelte index 4de44442a2..369582e7a2 100644 --- a/frontend/src/routes/(root)/+layout.svelte +++ b/frontend/src/routes/(root)/+layout.svelte @@ -4,6 +4,7 @@ import { page } from '$app/state' import { UserService, WorkspaceService } from '$lib/gen' import { logoutWithRedirect } from '$lib/logoutKit' + import { noteSessionEmail } from '$lib/onboardingProfile' import { clearWorkspaceFromStorage, userStore, @@ -161,6 +162,7 @@ ) } let user = await UserService.globalWhoami() + noteSessionEmail(user.email) console.log(`Welcome back ${user.email}`) } } catch (e) { diff --git a/frontend/src/routes/user/login_callback/[client_name]/+page.svelte b/frontend/src/routes/user/login_callback/[client_name]/+page.svelte index e45b43b871..b1e0f263a3 100644 --- a/frontend/src/routes/user/login_callback/[client_name]/+page.svelte +++ b/frontend/src/routes/user/login_callback/[client_name]/+page.svelte @@ -29,7 +29,29 @@ const rd = rawRd?.startsWith('http') && !isValidLogoutRedirect(rawRd) ? null : rawRd const closeUponLogin = getCookie('close') == 'true' || localStorage.getItem('closeUponLogin') == 'true' + // "Finish account setup" sent a signed-in account with no credentials of its own to a + // provider. Whatever went wrong on the way back — the consent screen cancelled, an + // address mismatch, an unverified address, a domain rule — that session is the only + // way into the account, so it must survive: report and go home rather than log out. + // Read before the backend call, which clears the cookie whether or not it adopts. + // SAML's ACS answers a top-level POST from the IdP, so a refusal there arrives here + // as a redirect with the flag in the query. + const finishingSetup = + !!getCookie('finish_setup') || page.url.searchParams.get('finish_setup') === '1' + function backToSetup(message: string) { + document.cookie = 'finish_setup=; path=/; max-age=0; SameSite=Lax' + sendUserToast(message, true) + goto('/') + } if (error) { + if (finishingSetup) { + backToSetup( + error.includes('finish_setup_mismatch') + ? error.replace(/^.*finish_setup_mismatch:\s*/, '') + : `Signing in with ${clientName} did not go through (${error}). Your account is unchanged.` + ) + return + } sendUserToast(`Error trying to login with ${clientName} ${error}`, true) if (closeUponLogin) { closeUponLoginError(`Error trying to login with ${clientName} ${error}`) @@ -40,6 +62,11 @@ try { await UserService.loginWithOauth({ requestBody: { code, state }, clientName }) } catch (e) { + const message = String(e?.body ?? e?.message ?? '') + if (finishingSetup) { + backToSetup(message.replace(/^.*finish_setup_mismatch:\s*/, '')) + return + } if (closeUponLogin) { closeUponLoginError(e.body ?? e.message) return diff --git a/frontend/src/routes/user/login_link_expired/+page.svelte b/frontend/src/routes/user/login_link_expired/+page.svelte new file mode 100644 index 0000000000..b160985bb2 --- /dev/null +++ b/frontend/src/routes/user/login_link_expired/+page.svelte @@ -0,0 +1,23 @@ + + + + + From a95e950529f6f01a220e09d322d5a4fb507ea694 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 14 Sep 2026 22:06:56 +0200 Subject: [PATCH 113/144] feat(cli): list, get and restore trashed items with wmill trash (#11125) * feat(cli): list, get and restore trashed items from the CLI * docs(cli): tell agents a sync push deletion is restorable with wmill trash * refactor(cli): share the ApiError formatting and type trash flags as integers --- backend/windmill-api/openapi.yaml | 147 ++++++++++++++++++ cli/src/commands/sync/sync.ts | 2 +- cli/src/commands/trash/trash.ts | 147 ++++++++++++++++++ cli/src/guidance/core.ts | 2 + cli/src/guidance/skills.gen.ts | 21 +++ cli/src/main.ts | 15 +- cli/src/utils/utils.ts | 17 ++ cli/test/trash_commands.test.ts | 66 ++++++++ .../lib/components/settings/Trashbin.svelte | 2 +- frontend/src/lib/services/trashService.ts | 69 -------- .../auto-generated/cli/cli-commands.md | 21 +++ system_prompts/auto-generated/prompts.ts | 21 +++ .../skills/cli-commands/SKILL.md | 21 +++ 13 files changed, 471 insertions(+), 80 deletions(-) create mode 100644 cli/src/commands/trash/trash.ts create mode 100644 cli/test/trash_commands.test.ts delete mode 100644 frontend/src/lib/services/trashService.ts diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 74c487dca1..21c059bc1a 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -293,6 +293,108 @@ paths: items: $ref: "#/components/schemas/AuditLog" + /w/{workspace}/trash/list: + get: + summary: list the workspace trashbin (requires admin privilege) + operationId: listTrash + tags: + - trash + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: item_kind + in: query + description: > + only return items of this kind: script, flow, app, schedule, variable, + resource, or a trigger kind such as http_trigger + schema: + type: string + - name: page + in: query + description: which page to return (starts at 0, default 0) + schema: + type: integer + - name: per_page + in: query + description: number of items to return for a given page (default 100, max 1000) + schema: + type: integer + responses: + "200": + description: the trashed items, most recently deleted first + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/TrashItem" + + /w/{workspace}/trash/get/{id}: + get: + summary: get a trashed item with the data it was deleted with (requires admin privilege) + operationId: getTrashItem + tags: + - trash + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/PathId" + responses: + "200": + description: the trashed item + content: + application/json: + schema: + $ref: "#/components/schemas/TrashItemWithData" + + /w/{workspace}/trash/restore/{id}: + post: + summary: restore a trashed item to its path (requires admin privilege) + operationId: restoreTrashItem + tags: + - trash + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/PathId" + responses: + "200": + description: item restored + content: + text/plain: + schema: + type: string + + /w/{workspace}/trash/delete/{id}: + delete: + summary: permanently delete a trashed item (requires admin privilege) + operationId: permanentlyDeleteTrashItem + tags: + - trash + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/PathId" + responses: + "200": + description: item permanently deleted + content: + text/plain: + schema: + type: string + + /w/{workspace}/trash/empty: + post: + summary: permanently delete every item in the workspace trashbin (requires admin privilege) + operationId: emptyTrash + tags: + - trash + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: trashbin emptied + content: + text/plain: + schema: + type: string + /auth/login: post: security: [] @@ -30015,6 +30117,51 @@ components: - operation - action_kind + TrashItem: + type: object + properties: + id: + type: integer + format: int64 + workspace_id: + type: string + item_kind: + type: string + description: script, flow, app, schedule, variable, resource, or a trigger kind such as http_trigger + item_path: + type: string + deleted_by: + type: string + deleted_at: + type: string + format: date-time + expires_at: + type: string + format: date-time + description: when the item is permanently deleted unless restored first + required: + - id + - workspace_id + - item_kind + - item_path + - deleted_by + - deleted_at + - expires_at + + TrashItemWithData: + allOf: + - $ref: "#/components/schemas/TrashItem" + - type: object + properties: + item_data: + type: object + additionalProperties: true + description: > + the deleted rows as they were stored; the shape depends on the kind, and a + secret variable's value stays encrypted + required: + - item_data + MainArgSignature: type: object properties: diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index 5f67d194fe..287dd6ae12 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -6611,7 +6611,7 @@ export async function push( if (deletedSecretBearing.length > 0) { log.info( colors.gray( - `${describeSecretBearingChanges(deletedSecretBearing)} deleted. The workspace trashbin keeps a deleted item for three days; a workspace admin can restore it from Workspace settings -> Trashbin.`, + `${describeSecretBearingChanges(deletedSecretBearing)} deleted. The workspace trashbin keeps a deleted item for three days; a workspace admin can restore it with \`wmill trash list\` and \`wmill trash restore \`, or from Workspace settings -> Trashbin.`, ), ); } diff --git a/cli/src/commands/trash/trash.ts b/cli/src/commands/trash/trash.ts new file mode 100644 index 0000000000..1d571e2884 --- /dev/null +++ b/cli/src/commands/trash/trash.ts @@ -0,0 +1,147 @@ +import { GlobalOptions } from "../../types.ts"; +import { requireLogin } from "../../core/auth.ts"; +import { resolveWorkspace } from "../../core/context.ts"; +import { Command } from "@cliffy/command"; +import { Table } from "@cliffy/table"; +import { colors } from "@cliffy/ansi/colors"; +import * as log from "../../core/log.ts"; +import { mergeConfigWithConfigFile } from "../../core/conf.ts"; +import * as wmill from "../../../gen/services.gen.ts"; +import { apiErrorMessage, formatTimestamp } from "../../utils/utils.ts"; + +async function list( + opts: GlobalOptions & { + json?: boolean; + kind?: string; + page?: number; + limit?: number; + } +) { + if (opts.json) log.setSilent(true); + opts = await mergeConfigWithConfigFile(opts); + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + + if (opts.page !== undefined && opts.page < 1) { + throw new Error("--page starts at 1"); + } + + const items = await wmill.listTrash({ + workspace: workspace.workspaceId, + itemKind: opts.kind, + // The trash endpoint counts pages from 0, unlike the API's other list + // endpoints whose `page` starts at 1; the flag counts from 1 like those. + page: opts.page === undefined ? undefined : opts.page - 1, + perPage: opts.limit, + }); + + if (opts.json) { + console.log(JSON.stringify(items)); + return; + } + if (items.length === 0) { + log.info("No trashed items found."); + return; + } + new Table() + .header(["ID", "Kind", "Path", "Deleted by", "Deleted at", "Expires at"]) + .padding(2) + .border(true) + .body( + items.map((item) => [ + String(item.id), + item.item_kind, + item.item_path, + item.deleted_by, + formatTimestamp(item.deleted_at), + formatTimestamp(item.expires_at), + ]) + ) + .render(); + log.info( + colors.gray( + "`wmill trash get ` shows what an item held, `wmill trash restore ` puts it back." + ) + ); +} + +async function get(opts: GlobalOptions & { json?: boolean }, id: number) { + if (opts.json) log.setSilent(true); + opts = await mergeConfigWithConfigFile(opts); + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + + const item = await wmill.getTrashItem({ + workspace: workspace.workspaceId, + id, + }); + + if (opts.json) { + console.log(JSON.stringify(item)); + return; + } + console.log(colors.bold("ID:") + " " + item.id); + console.log(colors.bold("Kind:") + " " + item.item_kind); + console.log(colors.bold("Path:") + " " + item.item_path); + console.log(colors.bold("Deleted by:") + " " + item.deleted_by); + console.log(colors.bold("Deleted at:") + " " + formatTimestamp(item.deleted_at)); + console.log(colors.bold("Expires at:") + " " + formatTimestamp(item.expires_at)); + console.log(colors.bold("Data:")); + console.log(JSON.stringify(item.item_data, null, 2)); +} + +async function restore(opts: GlobalOptions, ...ids: number[]) { + opts = await mergeConfigWithConfigFile(opts); + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + + let failed = 0; + for (const id of ids) { + try { + const message = await wmill.restoreTrashItem({ + workspace: workspace.workspaceId, + id, + }); + log.info(colors.green(message)); + } catch (e) { + failed += 1; + log.error( + `Could not restore trash item ${id}: ${apiErrorMessage(e) ?? String(e)}` + ); + } + } + if (failed > 0) { + process.exitCode = 1; + } +} + +const command = new Command() + .description( + "List, inspect and restore items deleted in the last three days (requires admin)" + ) + .option("--json", "Output as JSON (for piping to jq)") + .option( + "--kind ", + "Only items of this kind: script, flow, app, schedule, variable, resource or a trigger kind such as http_trigger" + ) + .option("--limit ", "Number of items to return (default 100, max 1000)") + .option("--page ", "Page to return, starting at 1") + .action(list as any) + .command("list", "List trashed items, most recently deleted first") + .option("--json", "Output as JSON (for piping to jq)") + .option( + "--kind ", + "Only items of this kind: script, flow, app, schedule, variable, resource or a trigger kind such as http_trigger" + ) + .option("--limit ", "Number of items to return (default 100, max 1000)") + .option("--page ", "Page to return, starting at 1") + .action(list as any) + .command("get", "Show a trashed item and the data it was deleted with") + .arguments("") + .option("--json", "Output as JSON (for piping to jq)") + .action(get as any) + .command("restore", "Put trashed items back at their paths") + .arguments("") + .action(restore as any); + +export default command; diff --git a/cli/src/guidance/core.ts b/cli/src/guidance/core.ts index 2a9bb34fe0..202a6feaad 100644 --- a/cli/src/guidance/core.ts +++ b/cli/src/guidance/core.ts @@ -165,6 +165,8 @@ No CI workflow runs \`wmill sync push\` automatically, so deploy directly from t - \`wmill sync push --dry-run\` to preview. - \`wmill sync push\` to apply. +A push deletes remote items that have no local file. They land in the workspace trashbin for three days: \`wmill trash list\` shows them and \`wmill trash restore \` puts one back (both need a workspace admin). + ### In both cases Only deploy when the user explicitly asks to deploy, publish, push, or ship — not when they say "run", "try", or "test". For testing local edits use the per-entity \`preview\` commands (\`wmill script preview\`, \`wmill flow preview\`) — they don't deploy. diff --git a/cli/src/guidance/skills.gen.ts b/cli/src/guidance/skills.gen.ts index b8b71b1de9..3f63574b83 100644 --- a/cli/src/guidance/skills.gen.ts +++ b/cli/src/guidance/skills.gen.ts @@ -7720,6 +7720,27 @@ Manage API tokens - \`--expiration \` - Token expiration (ISO 8601 timestamp) - \`token delete \` - Delete a token by its prefix +### trash + +List, inspect and restore items deleted in the last three days (requires admin) + +**Options:** +- \`--json\` - Output as JSON (for piping to jq) +- \`--kind \` - Only items of this kind: script, flow, app, schedule, variable, resource or a trigger kind such as http_trigger +- \`--limit \` - Number of items to return (default 100, max 1000) +- \`--page \` - Page to return, starting at 1 + +**Subcommands:** + +- \`trash list\` - List trashed items, most recently deleted first + - \`--json\` - Output as JSON (for piping to jq) + - \`--kind \` - Only items of this kind: script, flow, app, schedule, variable, resource or a trigger kind such as http_trigger + - \`--limit \` - Number of items to return (default 100, max 1000) + - \`--page \` - Page to return, starting at 1 +- \`trash get \` - Show a trashed item and the data it was deleted with + - \`--json\` - Output as JSON (for piping to jq) +- \`trash restore \` - Put trashed items back at their paths + ### trigger trigger related commands diff --git a/cli/src/main.ts b/cli/src/main.ts index ccce20da30..fa894b6f2d 100755 --- a/cli/src/main.ts +++ b/cli/src/main.ts @@ -29,7 +29,7 @@ import lint from "./commands/lint/lint.ts"; import dev from "./commands/dev/dev.ts"; import { GlobalOptions } from "./types.ts"; import { OpenAPI } from "../gen/index.ts"; -import { getHeaders } from "./utils/utils.ts"; +import { apiErrorMessage, getHeaders } from "./utils/utils.ts"; import { detectAuthGatewayChallenge } from "./utils/http_guards.ts"; import { setShowDiffs } from "./core/conf.ts"; import { markRequestsAsCliClient } from "./core/client.ts"; @@ -48,6 +48,7 @@ import job from "./commands/job/job.ts"; import group from "./commands/group/group.ts"; import audit from "./commands/audit/audit.ts"; import token from "./commands/token/token.ts"; +import trash from "./commands/trash/trash.ts"; import generateMetadata from "./commands/generate-metadata/generate-metadata.ts"; import docs from "./commands/docs/docs.ts"; import config from "./commands/config/config.ts"; @@ -214,6 +215,7 @@ const command = new Command() .command("group", group) .command("audit", audit) .command("token", token) + .command("trash", trash) .command("generate-metadata", generateMetadata) .command("docs", docs) .command("config", config) @@ -321,14 +323,9 @@ async function main() { await command.parse(args); } catch (e) { - if (e && typeof e === "object" && "name" in e && e.name === "ApiError") { - const body = (e as any).body; - let bodyStr = typeof body === "object" && body !== null ? JSON.stringify(body) : String(body ?? ""); - // Strip backend source file references like (flows.rs:1400) or @scripts.rs:123:45 - bodyStr = bodyStr.replace(/\s*[@(]\w+\.rs:\d+[:\d]*\)?/g, ""); - log.error( - "Server failed. " + (e as any).statusText + ": " + bodyStr - ); + const apiError = apiErrorMessage(e); + if (apiError !== undefined) { + log.error("Server failed. " + apiError); } else if (e instanceof Error) { log.error(e.message); } else if (e !== undefined && e !== null) { diff --git a/cli/src/utils/utils.ts b/cli/src/utils/utils.ts index 489a877af7..0801c03911 100644 --- a/cli/src/utils/utils.ts +++ b/cli/src/utils/utils.ts @@ -353,6 +353,23 @@ export function formatTimestamp(ts: string): string { return new Date(ts).toISOString().replace("T", " ").substring(0, 19); } +/** + * ": " for an error thrown by the generated API client, + * undefined for anything else. Backend source references such as + * `(flows.rs:1400)` are stripped from the body. + */ +export function apiErrorMessage(e: unknown): string | undefined { + if (!(e && typeof e === "object" && "name" in e && e.name === "ApiError")) { + return undefined; + } + const { body, statusText } = e as { body?: unknown; statusText?: string }; + const bodyStr = + typeof body === "object" && body !== null + ? JSON.stringify(body) + : String(body ?? ""); + return statusText + ": " + bodyStr.replace(/\s*[@(]\w+\.rs:\d+[:\d]*\)?/g, ""); +} + /** * Validate that required arguments are present when no -d data was provided. * Fetches the schema from the API and checks required fields. diff --git a/cli/test/trash_commands.test.ts b/cli/test/trash_commands.test.ts new file mode 100644 index 0000000000..7b417dbc5e --- /dev/null +++ b/cli/test/trash_commands.test.ts @@ -0,0 +1,66 @@ +import { expect, test, describe } from "bun:test"; +import { withTestBackend } from "./test_backend.ts"; +import { setupWorkspaceProfile, ensureFolder } from "./new_commands_helpers.ts"; + +describe("trash command", () => { + test("lists, shows and restores a deleted variable", async () => { + await withTestBackend(async (backend, tempDir) => { + await setupWorkspaceProfile(backend); + await ensureFolder(backend, "test"); + const ws = backend.workspace; + const path = `f/test/trash_${Date.now()}`; + const api = (route: string, init: RequestInit = {}) => + backend.apiRequest!(`/api/w/${ws}/${route}`, { + headers: { "Content-Type": "application/json" }, + ...init, + }); + + let resp = await api("variables/create", { + method: "POST", + body: JSON.stringify({ path, value: "kept", is_secret: false, description: "" }), + }); + expect(resp.status).toBeLessThan(300); + await resp.text(); + resp = await api(`variables/delete/${path}`, { method: "DELETE" }); + expect(resp.status).toBeLessThan(300); + await resp.text(); + + const list = await backend.runCLICommand( + ["trash", "list", "--json", "--kind", "variable"], + tempDir + ); + expect(list.code).toBe(0); + const item = JSON.parse(list.stdout).find((i: any) => i.item_path === path); + expect(item).toBeDefined(); + expect(item.item_kind).toBe("variable"); + + const get = await backend.runCLICommand( + ["trash", "get", "--json", String(item.id)], + tempDir + ); + expect(get.code).toBe(0); + expect(JSON.parse(get.stdout).item_data.row.value).toBe("kept"); + + // A bogus second id: the first restore must still go through, and the + // failure must show in the exit code. + const restore = await backend.runCLICommand( + ["trash", "restore", String(item.id), "999999999"], + tempDir + ); + expect(restore.code).toBe(1); + expect(restore.stdout).toContain(`variable '${path}' restored`); + expect(restore.stderr).toContain("999999999"); + + resp = await api(`variables/get/${path}`); + expect(resp.status).toBe(200); + expect((await resp.json()).value).toBe("kept"); + + const after = await backend.runCLICommand( + ["trash", "list", "--json", "--kind", "variable"], + tempDir + ); + expect(after.code).toBe(0); + expect(JSON.parse(after.stdout).some((i: any) => i.item_path === path)).toBe(false); + }); + }); +}); diff --git a/frontend/src/lib/components/settings/Trashbin.svelte b/frontend/src/lib/components/settings/Trashbin.svelte index 3b9b4f5f29..ce45e7a7e2 100644 --- a/frontend/src/lib/components/settings/Trashbin.svelte +++ b/frontend/src/lib/components/settings/Trashbin.svelte @@ -7,7 +7,7 @@ import Row from '$lib/components/table/Row.svelte' import { workspaceStore } from '$lib/stores' import { sendUserToast } from '$lib/toast' - import { type TrashItem, TrashService } from '$lib/services/trashService' + import { type TrashItem, TrashService } from '$lib/gen' import { Trash2, RotateCcw, diff --git a/frontend/src/lib/services/trashService.ts b/frontend/src/lib/services/trashService.ts deleted file mode 100644 index c1bdac5a23..0000000000 --- a/frontend/src/lib/services/trashService.ts +++ /dev/null @@ -1,69 +0,0 @@ -import { OpenAPI } from '$lib/gen/core/OpenAPI' -import { request as __request } from '$lib/gen/core/request' - -export type TrashItem = { - id: number - workspace_id: string - item_kind: string - item_path: string - deleted_by: string - deleted_at: string - expires_at: string -} - -export class TrashService { - public static listTrash(data: { - workspace: string - itemKind?: string - page?: number - perPage?: number - }): Promise { - return __request(OpenAPI, { - method: 'GET', - url: '/w/{workspace}/trash/list', - path: { - workspace: data.workspace - }, - query: { - item_kind: data.itemKind, - page: data.page, - per_page: data.perPage - } - }) - } - - public static restoreTrashItem(data: { workspace: string; id: number }): Promise { - return __request(OpenAPI, { - method: 'POST', - url: '/w/{workspace}/trash/restore/{id}', - path: { - workspace: data.workspace, - id: data.id - } - }) - } - - public static permanentlyDeleteTrashItem(data: { - workspace: string - id: number - }): Promise { - return __request(OpenAPI, { - method: 'DELETE', - url: '/w/{workspace}/trash/delete/{id}', - path: { - workspace: data.workspace, - id: data.id - } - }) - } - - public static emptyTrash(data: { workspace: string }): Promise { - return __request(OpenAPI, { - method: 'POST', - url: '/w/{workspace}/trash/empty', - path: { - workspace: data.workspace - } - }) - } -} diff --git a/system_prompts/auto-generated/cli/cli-commands.md b/system_prompts/auto-generated/cli/cli-commands.md index 833c039ea8..b18638ee23 100644 --- a/system_prompts/auto-generated/cli/cli-commands.md +++ b/system_prompts/auto-generated/cli/cli-commands.md @@ -668,6 +668,27 @@ Manage API tokens - `--expiration ` - Token expiration (ISO 8601 timestamp) - `token delete ` - Delete a token by its prefix +### trash + +List, inspect and restore items deleted in the last three days (requires admin) + +**Options:** +- `--json` - Output as JSON (for piping to jq) +- `--kind ` - Only items of this kind: script, flow, app, schedule, variable, resource or a trigger kind such as http_trigger +- `--limit ` - Number of items to return (default 100, max 1000) +- `--page ` - Page to return, starting at 1 + +**Subcommands:** + +- `trash list` - List trashed items, most recently deleted first + - `--json` - Output as JSON (for piping to jq) + - `--kind ` - Only items of this kind: script, flow, app, schedule, variable, resource or a trigger kind such as http_trigger + - `--limit ` - Number of items to return (default 100, max 1000) + - `--page ` - Page to return, starting at 1 +- `trash get ` - Show a trashed item and the data it was deleted with + - `--json` - Output as JSON (for piping to jq) +- `trash restore ` - Put trashed items back at their paths + ### trigger trigger related commands diff --git a/system_prompts/auto-generated/prompts.ts b/system_prompts/auto-generated/prompts.ts index dc27234d60..216c72c128 100644 --- a/system_prompts/auto-generated/prompts.ts +++ b/system_prompts/auto-generated/prompts.ts @@ -3864,6 +3864,27 @@ Manage API tokens - \`--expiration \` - Token expiration (ISO 8601 timestamp) - \`token delete \` - Delete a token by its prefix +### trash + +List, inspect and restore items deleted in the last three days (requires admin) + +**Options:** +- \`--json\` - Output as JSON (for piping to jq) +- \`--kind \` - Only items of this kind: script, flow, app, schedule, variable, resource or a trigger kind such as http_trigger +- \`--limit \` - Number of items to return (default 100, max 1000) +- \`--page \` - Page to return, starting at 1 + +**Subcommands:** + +- \`trash list\` - List trashed items, most recently deleted first + - \`--json\` - Output as JSON (for piping to jq) + - \`--kind \` - Only items of this kind: script, flow, app, schedule, variable, resource or a trigger kind such as http_trigger + - \`--limit \` - Number of items to return (default 100, max 1000) + - \`--page \` - Page to return, starting at 1 +- \`trash get \` - Show a trashed item and the data it was deleted with + - \`--json\` - Output as JSON (for piping to jq) +- \`trash restore \` - Put trashed items back at their paths + ### trigger trigger related commands diff --git a/system_prompts/auto-generated/skills/cli-commands/SKILL.md b/system_prompts/auto-generated/skills/cli-commands/SKILL.md index e562938f48..3aafa0be85 100644 --- a/system_prompts/auto-generated/skills/cli-commands/SKILL.md +++ b/system_prompts/auto-generated/skills/cli-commands/SKILL.md @@ -673,6 +673,27 @@ Manage API tokens - `--expiration ` - Token expiration (ISO 8601 timestamp) - `token delete ` - Delete a token by its prefix +### trash + +List, inspect and restore items deleted in the last three days (requires admin) + +**Options:** +- `--json` - Output as JSON (for piping to jq) +- `--kind ` - Only items of this kind: script, flow, app, schedule, variable, resource or a trigger kind such as http_trigger +- `--limit ` - Number of items to return (default 100, max 1000) +- `--page ` - Page to return, starting at 1 + +**Subcommands:** + +- `trash list` - List trashed items, most recently deleted first + - `--json` - Output as JSON (for piping to jq) + - `--kind ` - Only items of this kind: script, flow, app, schedule, variable, resource or a trigger kind such as http_trigger + - `--limit ` - Number of items to return (default 100, max 1000) + - `--page ` - Page to return, starting at 1 +- `trash get ` - Show a trashed item and the data it was deleted with + - `--json` - Output as JSON (for piping to jq) +- `trash restore ` - Put trashed items back at their paths + ### trigger trigger related commands From d0cac0807f1b6f4fc76d6e5f7e27e03d30abec8d Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 14 Sep 2026 22:15:37 +0200 Subject: [PATCH 114/144] fix: set the enclosing span's trace context on exported log records (#11123) * chore: pin the EE ref that stamps trace context on exported log records Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01WfgEm5rNRDyw4WUYf9ToVz * chore: pin the EE ref with the sampling-decision test Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01WfgEm5rNRDyw4WUYf9ToVz * chore: update ee-repo-ref to 04a9f1efb4a52c79fcd20258b34780c86103d27f This commit updates the EE repository reference after PR #800 was merged in windmill-ee-private. Previous ee-repo-ref: d48361d66580618cb7a934d3c5f56a0c7e39ffaa New ee-repo-ref: 04a9f1efb4a52c79fcd20258b34780c86103d27f Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Fable 5.1 Co-authored-by: windmill-internal-app[bot] --- backend/ee-repo-ref.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 062211f925..a02a129b97 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -1ba6fe83451f0a1f8fafe04b7187087d51e0f769 +04a9f1efb4a52c79fcd20258b34780c86103d27f From e8c02c04cdb1a3f199f3f0a8b53a839a53d4a1d9 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 14 Sep 2026 22:32:45 +0200 Subject: [PATCH 115/144] feat: windmill-chat sdk for chat-mode flows in external frontends and raw apps (#11117) * feat: windmill-chat sdk for chat-mode flows in external frontends and raw apps Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_018aQiZNAU8g17kWkyTryS5J * fix: keep streamed answers until persisted, finish turns after history fallback Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_018aQiZNAU8g17kWkyTryS5J * feat: ai sdk transport and assistant-ui runtime for windmill-chat Co-Authored-By: Claude Fable 5.1 * fix: finish a turn from the flow result until its answer row lands, hash chat ids without crypto.subtle Co-Authored-By: Claude Fable 5.1 * fix: judge a turn answered by a persisted assistant row, wherever it was fetched Co-Authored-By: Claude Fable 5.1 * fix: attribute a turn's answer to its own jobs, keep a local turn when switching conversations Co-Authored-By: Claude Fable 5.1 * fix: mirror local history on every change, attribute failure-handler answers to the turn Co-Authored-By: Claude Fable 5.1 * fix: new chat per token string in the React hook, idle after destroy, no reorder on view Co-Authored-By: Claude Fable 5.1 * fix: recreate the hook's chat on any credential change, namespace local history per user Co-Authored-By: Claude Fable 5.1 * fix: send the latest inputs from the React hook Co-Authored-By: Claude Fable 5.1 --------- Co-authored-by: Claude Fable 5.1 --- .github/change-versions-mac.sh | 1 + .github/change-versions.sh | 3 + .github/workflows/npm_on_release.yml | 11 + .github/workflows/sdk-tests.yml | 19 + backend/windmill-api/openapi.yaml | 3 +- backend/windmill-api/src/apps.rs | 8 +- backend/windmill-api/src/lib.rs | 6 +- .../src/mcp/auto_generated_endpoints.rs | 16 +- chat-sdk/.gitignore | 2 + chat-sdk/README.md | 283 ++ chat-sdk/package-lock.json | 3451 +++++++++++++++++ chat-sdk/package.json | 108 + chat-sdk/src/ai-sdk.ts | 324 ++ chat-sdk/src/api.ts | 294 ++ chat-sdk/src/assistant-ui.ts | 124 + chat-sdk/src/chat.ts | 744 ++++ chat-sdk/src/config.ts | 81 + chat-sdk/src/follow.ts | 54 + chat-sdk/src/history.ts | 90 + chat-sdk/src/index.ts | 29 + chat-sdk/src/react.ts | 67 + chat-sdk/src/stream.ts | 65 + chat-sdk/src/types.ts | 123 + chat-sdk/src/utils.ts | 134 + chat-sdk/test/ai-sdk-chat.test.ts | 62 + chat-sdk/test/ai-sdk.test.ts | 178 + chat-sdk/test/assistant-ui.test.ts | 38 + chat-sdk/test/chat.test.ts | 671 ++++ chat-sdk/test/config.test.ts | 49 + chat-sdk/test/react.test.tsx | 80 + chat-sdk/test/stream.test.ts | 61 + chat-sdk/test/support.ts | 101 + chat-sdk/tsconfig.build.json | 12 + chat-sdk/tsconfig.json | 15 + chat-sdk/tsdown.config.ts | 9 + cli/src/guidance/skills.gen.ts | 14 + .../src/lib/components/raw_apps/sdkScopes.ts | 12 +- frontend/src/lib/mcpEndpointTools.ts | 16 +- system_prompts/auto-generated/prompts.ts | 14 + .../auto-generated/skills/raw-app/SKILL.md | 14 + system_prompts/base/raw-app.md | 14 + 41 files changed, 7384 insertions(+), 16 deletions(-) create mode 100644 chat-sdk/.gitignore create mode 100644 chat-sdk/README.md create mode 100644 chat-sdk/package-lock.json create mode 100644 chat-sdk/package.json create mode 100644 chat-sdk/src/ai-sdk.ts create mode 100644 chat-sdk/src/api.ts create mode 100644 chat-sdk/src/assistant-ui.ts create mode 100644 chat-sdk/src/chat.ts create mode 100644 chat-sdk/src/config.ts create mode 100644 chat-sdk/src/follow.ts create mode 100644 chat-sdk/src/history.ts create mode 100644 chat-sdk/src/index.ts create mode 100644 chat-sdk/src/react.ts create mode 100644 chat-sdk/src/stream.ts create mode 100644 chat-sdk/src/types.ts create mode 100644 chat-sdk/src/utils.ts create mode 100644 chat-sdk/test/ai-sdk-chat.test.ts create mode 100644 chat-sdk/test/ai-sdk.test.ts create mode 100644 chat-sdk/test/assistant-ui.test.ts create mode 100644 chat-sdk/test/chat.test.ts create mode 100644 chat-sdk/test/config.test.ts create mode 100644 chat-sdk/test/react.test.tsx create mode 100644 chat-sdk/test/stream.test.ts create mode 100644 chat-sdk/test/support.ts create mode 100644 chat-sdk/tsconfig.build.json create mode 100644 chat-sdk/tsconfig.json create mode 100644 chat-sdk/tsdown.config.ts diff --git a/.github/change-versions-mac.sh b/.github/change-versions-mac.sh index 2f23b21eb8..72bb2b9d8c 100755 --- a/.github/change-versions-mac.sh +++ b/.github/change-versions-mac.sh @@ -12,6 +12,7 @@ sed -i '' -e "/^export const VERSION =/s/= .*/= \"v$VERSION\";/" ${root_dirpath} sed -i '' -e "/version: /s/: .*/: $VERSION/" ${root_dirpath}/backend/windmill-api/openapi.yaml sed -i '' -e "/version: /s/: .*/: $VERSION/" ${root_dirpath}/openflow.openapi.yaml sed -i '' -e "/\"version\": /s/: .*,/: \"$VERSION\",/" ${root_dirpath}/typescript-client/package.json +sed -i '' -e "/\"version\": /s/: .*,/: \"$VERSION\",/" ${root_dirpath}/chat-sdk/package.json sed -i '' -e "/\"version\": /s/: .*,/: \"$VERSION\",/" ${root_dirpath}/frontend/package.json sed -i '' -e "/^version =/s/= .*/= \"$VERSION\"/" ${root_dirpath}/python-client/wmill/pyproject.toml sed -i '' -e "/^windmill-api =/s/= .*/= \"\\^$VERSION\"/" ${root_dirpath}/python-client/wmill/pyproject.toml diff --git a/.github/change-versions.sh b/.github/change-versions.sh index ffb73f6180..1772ddbc89 100755 --- a/.github/change-versions.sh +++ b/.github/change-versions.sh @@ -13,6 +13,7 @@ sed -i -e "/version: /s/: .*/: $VERSION/" ${root_dirpath}/backend/windmill-api/o sed -i -e "/version: /s/: .*/: $VERSION/" ${root_dirpath}/openflow.openapi.yaml sed -i -e "/\"version\": /s/: .*,/: \"$VERSION\",/" ${root_dirpath}/typescript-client/package.json sed -i -e "/\"version\": /s/: .*,/: \"$VERSION\",/" ${root_dirpath}/typescript-client/jsr.json +sed -i -e "/\"version\": /s/: .*,/: \"$VERSION\",/" ${root_dirpath}/chat-sdk/package.json sed -i -e "/\"version\": /s/: .*,/: \"$VERSION\",/" ${root_dirpath}/frontend/package.json sed -i -e "/\"version\": /s/: .*,/: \"$VERSION\",/" ${root_dirpath}/windmill-yaml-validator/package.json sed -i -e "/^version =/s/= .*/= \"$VERSION\"/" ${root_dirpath}/python-client/wmill/pyproject.toml @@ -33,3 +34,5 @@ cd ${root_dirpath}/frontend && npm i --package-lock-only --ignore-scripts # The CLI installs this package on every `bun install`, which would otherwise rewrite the # lockfile's version and leave a dirty tree. cd ${root_dirpath}/windmill-yaml-validator && npm i --package-lock-only --ignore-scripts + +cd ${root_dirpath}/chat-sdk && npm i --package-lock-only --ignore-scripts diff --git a/.github/workflows/npm_on_release.yml b/.github/workflows/npm_on_release.yml index a41bd80854..7d30265c74 100644 --- a/.github/workflows/npm_on_release.yml +++ b/.github/workflows/npm_on_release.yml @@ -17,6 +17,17 @@ jobs: - run: cd typescript-client && ./publish.sh --access public && cd .. env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + publish_chat_sdk: + runs-on: ubicloud-standard-8 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v3 + with: + node-version: "20.x" + registry-url: "https://registry.npmjs.org" + - run: cd chat-sdk && npm ci && npm run build && npm publish --access public + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} publish_cli: runs-on: ubicloud-standard-8 steps: diff --git a/.github/workflows/sdk-tests.yml b/.github/workflows/sdk-tests.yml index efcbcd6ec1..6f7d79c6af 100644 --- a/.github/workflows/sdk-tests.yml +++ b/.github/workflows/sdk-tests.yml @@ -32,6 +32,25 @@ jobs: working-directory: ./typescript-client run: bun test --timeout 120000 tests/ + chat-sdk: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Bun + uses: oven-sh/setup-bun@v2 + with: + bun-version: latest + + - uses: actions/setup-node@v4 + with: + node-version: "20.x" + + - name: Run tests + working-directory: ./chat-sdk + run: npm ci && npm run check && bun test + python-client: runs-on: ubuntu-latest steps: diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 21c059bc1a..aee198e92a 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -34334,7 +34334,8 @@ components: carrying their own identity restricted to these scopes, handed to the app bundle so `windmill-client` calls run as the viewer. Must be a subset of the server's curated allowlist (jobs:run, jobs:read, - users:read, resources:read, variables:read). + users:read, resources:read, variables:read, flow_conversations:read, + flow_conversations:write). ListableApp: type: object diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index daa7fee59b..cc4c5079f9 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -1440,12 +1440,14 @@ const APP_EMBED_TOKEN_VALIDITY_HOURS: i64 = 12; /// Scopes an app author may declare in `Policy::frontend_sdk_scopes`. No `apps:*` /// scope, so the token cannot reach the mint endpoints and renew itself; the /// `raw_app_sdk` sentinel narrows the rest (see `scopes.rs`). -pub const FRONTEND_SDK_ALLOWED_SCOPES: [&str; 5] = [ +pub const FRONTEND_SDK_ALLOWED_SCOPES: [&str; 7] = [ "jobs:run", "jobs:read", "users:read", "resources:read", "variables:read", + "flow_conversations:read", + "flow_conversations:write", ]; /// Reject a policy declaring frontend SDK scopes outside the curated list. @@ -6014,6 +6016,10 @@ mod embed_token_tests { // the author declared it and the viewer consented. ("/api/w/test/resources/get_value/u/admin/r", "GET"), ("/api/w/test/variables/get_value/u/admin/v", "GET"), + // A chat UI's history for a chat-mode flow; RLS keeps it to the viewer's own. + ("/api/w/test/flow_conversations/list", "GET"), + ("/api/w/test/flow_conversations/some-uuid/messages", "GET"), + ("/api/w/test/flow_conversations/delete/some-uuid", "DELETE"), ]; for (path, method) in allowed { assert!( diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index fc2c773703..bd6fda015b 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -659,9 +659,13 @@ pub async fn run_server( "/workspace_dependencies", workspace_dependencies::workspaced_service(), ) + // CORS so a chat UI on another origin (an external site, or + // a sandboxed raw app with its frontend SDK token) can read + // its conversation history. Bearer-only, like variables. .nest( "/flow_conversations", - windmill_api_flow_conversations::workspaced_service(), + windmill_api_flow_conversations::workspaced_service() + .layer(cors.clone()), ) // CORS so an opaque-origin app iframe (WIN-2006 embed, // no separate domain) can read folders/listnames with a diff --git a/backend/windmill-api/src/mcp/auto_generated_endpoints.rs b/backend/windmill-api/src/mcp/auto_generated_endpoints.rs index ea673610e6..ffefaba2dd 100644 --- a/backend/windmill-api/src/mcp/auto_generated_endpoints.rs +++ b/backend/windmill-api/src/mcp/auto_generated_endpoints.rs @@ -1268,10 +1268,12 @@ is, a different one moves it there and archives the old path"), "description": "Who may open the app, and who its runnables execute as. Optional, and what omitting it means depends on the operation: creating an app defaults it to `publisher` (runs on behalf of the app's publisher and requires an authenticated viewer), while updating one keeps the mode the app is already deployed under. Neither `anonymous`, which makes the app publicly executable, nor `guest`, which opens it to anyone the identity provider authenticates, is ever assumed. A guest is only admitted where the workspace also has `guest_access_enabled`, which is checked when the session is minted and again on every guest request. Possible values: viewer, publisher, guest, anonymous" }, "on_behalf_of": { - "type": "string" + "type": "string", + "description": "The user or group the app runs as in anonymous or publisher mode (e.g. 'u/admin' or 'g/mygroup'). The authority for the app's identity." }, "on_behalf_of_email": { - "type": "string" + "type": "string", + "description": "Address of `on_behalf_of`, written through from it on every save and returned as stored. Optional; when absent it is derived from `on_behalf_of`. Sending it is optional too; it must name the same account as `on_behalf_of`, and a pair that disagrees is rejected." }, "sandbox": { "type": "boolean", @@ -1282,7 +1284,7 @@ is, a different one moves it there and archives the old path"), "items": { "type": "string" }, - "description": "Raw apps: author-declared scopes for the frontend SDK token. Takes effect only when `sandbox` is also true — an unsandboxed bundle runs with the viewer's own session, so no token is advertised or minted for it and this list stays inert. On a sandboxed app a non-empty list lets viewers mint (after consenting) a short-lived token carrying their own identity restricted to these scopes, handed to the app bundle so `windmill-client` calls run as the viewer. Must be a subset of the server's curated allowlist (jobs:run, jobs:read, users:read, resources:read, variables:read).\n" + "description": "Raw apps: author-declared scopes for the frontend SDK token. Takes effect only when `sandbox` is also true — an unsandboxed bundle runs with the viewer's own session, so no token is advertised or minted for it and this list stays inert. On a sandboxed app a non-empty list lets viewers mint (after consenting) a short-lived token carrying their own identity restricted to these scopes, handed to the app bundle so `windmill-client` calls run as the viewer. Must be a subset of the server's curated allowlist (jobs:run, jobs:read, users:read, resources:read, variables:read, flow_conversations:read, flow_conversations:write).\n" } } } @@ -1383,10 +1385,12 @@ is, a different one moves it there and archives the old path"), "description": "Who may open the app, and who its runnables execute as. Optional, and what omitting it means depends on the operation: creating an app defaults it to `publisher` (runs on behalf of the app's publisher and requires an authenticated viewer), while updating one keeps the mode the app is already deployed under. Neither `anonymous`, which makes the app publicly executable, nor `guest`, which opens it to anyone the identity provider authenticates, is ever assumed. A guest is only admitted where the workspace also has `guest_access_enabled`, which is checked when the session is minted and again on every guest request. Possible values: viewer, publisher, guest, anonymous" }, "on_behalf_of": { - "type": "string" + "type": "string", + "description": "The user or group the app runs as in anonymous or publisher mode (e.g. 'u/admin' or 'g/mygroup'). The authority for the app's identity." }, "on_behalf_of_email": { - "type": "string" + "type": "string", + "description": "Address of `on_behalf_of`, written through from it on every save and returned as stored. Optional; when absent it is derived from `on_behalf_of`. Sending it is optional too; it must name the same account as `on_behalf_of`, and a pair that disagrees is rejected." }, "sandbox": { "type": "boolean", @@ -1397,7 +1401,7 @@ is, a different one moves it there and archives the old path"), "items": { "type": "string" }, - "description": "Raw apps: author-declared scopes for the frontend SDK token. Takes effect only when `sandbox` is also true — an unsandboxed bundle runs with the viewer's own session, so no token is advertised or minted for it and this list stays inert. On a sandboxed app a non-empty list lets viewers mint (after consenting) a short-lived token carrying their own identity restricted to these scopes, handed to the app bundle so `windmill-client` calls run as the viewer. Must be a subset of the server's curated allowlist (jobs:run, jobs:read, users:read, resources:read, variables:read).\n" + "description": "Raw apps: author-declared scopes for the frontend SDK token. Takes effect only when `sandbox` is also true — an unsandboxed bundle runs with the viewer's own session, so no token is advertised or minted for it and this list stays inert. On a sandboxed app a non-empty list lets viewers mint (after consenting) a short-lived token carrying their own identity restricted to these scopes, handed to the app bundle so `windmill-client` calls run as the viewer. Must be a subset of the server's curated allowlist (jobs:run, jobs:read, users:read, resources:read, variables:read, flow_conversations:read, flow_conversations:write).\n" } } }, diff --git a/chat-sdk/.gitignore b/chat-sdk/.gitignore new file mode 100644 index 0000000000..1eae0cf670 --- /dev/null +++ b/chat-sdk/.gitignore @@ -0,0 +1,2 @@ +dist/ +node_modules/ diff --git a/chat-sdk/README.md b/chat-sdk/README.md new file mode 100644 index 0000000000..98874a719f --- /dev/null +++ b/chat-sdk/README.md @@ -0,0 +1,283 @@ +# windmill-chat + +Build a chat interface on a Windmill flow deployed in **chat mode**, from any frontend +or from a Windmill raw app. The library is headless: it runs the flow, follows the +answer as it streams, keeps the conversation history, and hands you state to render. + +``` +npm install windmill-chat +``` + +No runtime dependencies. Optional peers: `react` for `windmill-chat/react`, `ai` for +`windmill-chat/ai-sdk`, `@assistant-ui/react` for `windmill-chat/assistant-ui`. + +| You build the UI with | Import | You get | +|---|---|---| +| Vercel AI SDK `useChat`, AI Elements | `windmill-chat/ai-sdk` | a `ChatTransport`: `useChat({ transport })`, nothing else changes | +| assistant-ui | `windmill-chat/assistant-ui` | a runtime for `AssistantRuntimeProvider`, threads included | +| Your own components | `windmill-chat/react` or `windmill-chat` | a hook / a store with messages, status and actions | + +## The flow + +Any deployed flow with **Chat mode** enabled in its settings works. Windmill passes the +message as the `user_message` input and threads the conversation through `memory_id`, +so an AI agent step remembers earlier turns. The answer is: + +- what the last step streams, when it is an AI agent step; +- otherwise the flow's result: its `windmill_chat_answer` field when it has one, a + string as is, anything else as JSON. + +## Vercel AI SDK (`useChat`, AI Elements) + +```tsx +import { useChat } from '@ai-sdk/react' +import { createWindmillChatTransport } from 'windmill-chat/ai-sdk' + +const transport = createWindmillChatTransport({ + baseUrl: 'https://app.windmill.dev', + workspace: 'acme', + flowPath: 'f/support/assistant', + token: () => fetch('/api/windmill-token').then((r) => r.text()) +}) + +export function Support() { + const { messages, status, sendMessage, stop } = useChat({ id: conversationId, transport }) + // render `messages[i].parts`: text, reasoning and dynamic-tool parts, as with any AI SDK backend +} +``` + +The chat `id` is the conversation: reuse it to continue one, and pass a UUID when you +also read server history, so it matches what `flow_conversations` stores (any other id +maps to a fixed UUID). `sendMessage(msg, { body })` sends extra flow inputs. Tool calls +arrive as `dynamic-tool` parts (`input-available → output-available | output-error`), +which AI Elements' `` renders as is. A failed flow surfaces as `error`. +`regenerate()` runs the flow again with the same message: a new turn on the server, +not a replacement of the previous answer. + +The transport also carries the history helpers: `transport.loadMessages(id)` returns +`UIMessage`s for `useChat({ messages })` or `setMessages`, `transport.listConversations()` +and `transport.deleteConversation(id)`. Attachments are not supported: `sendMessage` with +`files` is refused with an explanatory error. + +## assistant-ui + +```tsx +import { AssistantRuntimeProvider } from '@assistant-ui/react' +import { useWindmillRuntime } from 'windmill-chat/assistant-ui' + +export function Support() { + const runtime = useWindmillRuntime({ baseUrl, workspace, flowPath, token }) + return ( + + {/* your assistant-ui components, thread list included */} + + ) +} +``` + +Conversations are threads: `ThreadListPrimitive` switches, creates and deletes them. +Tool calls render through your `tools` components (`MessagePrimitive.Parts`), reasoning +through `Reasoning`. It takes the same options as `useWindmillChat` below. + +## React + +```tsx +import { useWindmillChat } from 'windmill-chat/react' + +export function Support() { + const chat = useWindmillChat({ + baseUrl: 'https://app.windmill.dev', + workspace: 'acme', + flowPath: 'f/support/assistant', + token: () => fetch('/api/windmill-token').then((r) => r.text()) + }) + const [draft, setDraft] = useState('') + + return ( +
    + {chat.messages.map((m) => ( +

    + {m.content} +

    + ))} +
    { + e.preventDefault() + chat.sendMessage(draft) + setDraft('') + }} + > + setDraft(e.target.value)} /> + + {chat.status === 'streaming' && ( + + )} + +
    + ) +} +``` + +The hook returns the [state](#state) plus the chat's methods. It recreates the chat +(fresh state, old one destroyed) when `flowPath`, `baseUrl`, `workspace`, `history`, +`storageKey` or the credential change: a different token string, or a switch between +no token, a string and a function. A token function is called through a ref, so +passing a new closure on every render is fine and never resets the chat; when users +sign in and out behind a token function, change `storageKey` (their id) so local +history and state start over with them. + +## Raw apps + +Inside a Windmill raw app nothing needs configuring: the chat runs as the viewer, +against the Windmill the app is served from. + +```tsx +const chat = useWindmillChat({ flowPath: 'f/support/assistant' }) +``` + +- **Unsandboxed app** (the default): the viewer's session is used. Viewers need + permission to run the flow. +- **Sandboxed app**: declare `jobs:run` in the app's frontend SDK scopes, and + `flow_conversations:write` for server-side history. The viewer consents once and + the app receives a token restricted to those scopes. +- **`wmill app dev`**: there is no viewer session on the dev server, so pass + `baseUrl`, `workspace` and `token` explicitly during development. + +## Any framework + +`createChat` returns a store: `subscribe` calls the listener immediately and on every +change, and returns the unsubscribe function. That is the Svelte store contract, so +`$chat` works as is; other frameworks wrap it in a few lines. + +```ts +import { createChat } from 'windmill-chat' + +const chat = createChat({ baseUrl, workspace, flowPath, token }) +chat.subscribe((state) => render(state)) +await chat.sendMessage('Hello') +``` + +```svelte + + +{#each $chat.messages as m (m.id)} +

    {m.content}

    +{/each} + +``` + +## Options + +| Option | | +|---|---| +| `flowPath` | Path of the deployed flow, e.g. `f/support/assistant`. Required. | +| `baseUrl` | The Windmill origin. Detected inside a raw app. | +| `workspace` | Detected inside a raw app. | +| `token` | A token, or a function returning one (called before every request, so it can fetch a short-lived token from your backend). Omit it inside a raw app. | +| `history` | `'server'`, `'local'` or `'none'`, see [History](#history). Defaults to `'server'` with a viewer session and `'local'` with an explicit `token`. | +| `inputs` | Extra flow inputs sent with every message. `sendMessage(text, { inputs })` adds per-message ones. | +| `storageKey` | Namespace for `local` history, e.g. the signed-in user's id. Local history is per browser and per flow; without it, users sharing a browser share it. | +| `fetch`, `storage` | Replacements for the globals, for tests and unusual runtimes. | +| `pageSize` | Messages and conversations per page of server history. Default 50. | +| `onFinish`, `onError` | Called when a turn has its answer, or could not run at all. | + +## State + +```ts +interface ChatState { + conversationId: string | undefined + messages: ChatMessage[] + status: 'idle' | 'submitted' | 'streaming' | 'error' + error: Error | undefined + conversations: Conversation[] + history: 'server' | 'local' | 'none' + loadingMessages: boolean + hasMoreMessages: boolean +} + +interface ChatMessage { + id: string + role: 'user' | 'assistant' | 'tool' | 'system' + content: string + reasoning?: string // the model's reasoning summary, when streamed + tool?: { callId?: string; name: string; arguments?: string; result?: string; status: 'running' | 'success' | 'error' } + success: boolean // false for a failed flow or tool + pending: boolean // still streaming, or not yet confirmed by the server + createdAt: string + jobId?: string + stepName?: string + serverId?: string // the persisted row; `id` itself never changes, so list keys are stable +} +``` + +A turn goes `submitted` (the flow is queued) → `streaming` (the answer is arriving) → +`idle`. Tool calls appear as `tool` messages whose `status` moves from `running` to +`success` or `error`. A flow that fails still completes the turn: its error is the +answer, an `assistant` message with `success: false`. `status: 'error'` (with `error` +set) means the turn could not run or be followed at all, such as a refused request. + +Methods: `sendMessage(text, { inputs? })`, `stop()`, `newConversation()`, +`selectConversation(id)`, `loadConversations({ page?, perPage? })`, +`deleteConversation(id)`, `loadOlderMessages()`, `destroy()`. Switching conversations +stops following the current answer; the flow keeps running and, with server history, +its answer is there when you come back. + +## History + +Windmill stores every conversation of a chat-mode flow, and each Windmill user sees +only their own. `history: 'server'` reads that store: `loadConversations()` lists +them, `selectConversation(id)` loads one, `loadOlderMessages()` pages back. Every +message rendered from the server carries its `jobId` and `stepName`. + +That store is keyed by the **Windmill user**, so it fits a viewer session or a token +issued per user. With one token shared by every visitor of a site, all visitors would +see each other's conversations. For that setup use `history: 'local'` (the default +with an explicit `token`): the conversation list and messages stay in the browser's +`localStorage`, per Windmill instance, workspace and flow. `'none'` keeps nothing +beyond the page. + +When the default `'server'` mode turns out unreadable (a token or sandboxed app +without `flow_conversations` scopes), the chat switches itself to `'local'` and +`state.history` says so. Passing `history` explicitly disables that fallback. + +## Tokens + +Anything a browser holds can be read by its user, so give a chat token exactly what +the chat needs: + +| Setup | Scopes | +|---|---| +| Public site, one token for everyone | `jobs:run:flows:f/support/assistant`, and `history: 'local'`. The token can run that one flow and follow its jobs, nothing else. | +| Per-user tokens minted by your backend | The above plus `flow_conversations:write`, with `history: 'server'` (an explicit token defaults to local history). Return them from an endpoint and pass `token: () => fetch(...)`. | +| A Windmill user in the browser (raw app, embedded Windmill) | No token: the session is used. | + +The token's user must be allowed to run the flow. `stop()` closes the stream in any +case; cancelling the run on the server as well needs `jobs:write`, which also lets the +token read every job its user can see, so leave it out unless that matters. + +Anyone holding the token can run the flow with inputs of their choosing, so a flow +exposed this way should treat `user_message` and the other inputs as untrusted. + +## Lower level + +`WindmillChatApi` wraps the endpoints (`runFlow`, `streamJob`, `listConversations`, +`listMessages`, `deleteConversation`, `cancelJob`), `followJob` follows a run to +completion across the server's stream timeouts, `parseStreamEvents` decodes the AI +agent stream, `extractChatAnswer` turns a flow result into the text a chat shows, and +`conversationIdFor` maps any chat id to its conversation UUID. They are exported for +custom integrations. + +## For AI coding agents + +When asked to add a chat over a Windmill flow: the flow must be deployed with chat mode +on. Pick the entry point from the table at the top (`useChat` → `windmill-chat/ai-sdk`, +assistant-ui → `windmill-chat/assistant-ui`, otherwise `windmill-chat/react`). Inside a +Windmill raw app pass only `flowPath`. Elsewhere pass `baseUrl`, `workspace` and a +`token`; for a public page use a token scoped to `jobs:run:flows:` and leave +`history` at its default. Render `role`, `content`, `pending`, `success` and +`tool.status`; never build the SSE handling yourself. diff --git a/chat-sdk/package-lock.json b/chat-sdk/package-lock.json new file mode 100644 index 0000000000..7d4ff28c21 --- /dev/null +++ b/chat-sdk/package-lock.json @@ -0,0 +1,3451 @@ +{ + "name": "windmill-chat", + "version": "1.810.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "windmill-chat", + "version": "1.810.0", + "license": "Apache-2.0", + "devDependencies": { + "@ai-sdk/react": "^4.0.102", + "@assistant-ui/react": "^0.15.19", + "@happy-dom/global-registrator": "^20.14.5", + "@types/bun": "^1.3.0", + "@types/react": "^19.0.0", + "@types/react-dom": "^19.3.0", + "ai": "^7.0.99", + "react": "^19.0.0", + "react-dom": "^19.3.0", + "tsdown": "^0.12.9", + "typescript": "^5.4.5" + }, + "peerDependencies": { + "@assistant-ui/react": ">=0.15", + "ai": ">=5", + "react": ">=18" + }, + "peerDependenciesMeta": { + "@assistant-ui/react": { + "optional": true + }, + "ai": { + "optional": true + }, + "react": { + "optional": true + } + } + }, + "node_modules/@ai-sdk/gateway": { + "version": "4.0.80", + "resolved": "https://registry.npmjs.org/@ai-sdk/gateway/-/gateway-4.0.80.tgz", + "integrity": "sha512-6t+07o8lSthpKf64Xb1qHWR2bWvJ3Fd2oFvS9fQc45p31bi2OUqan246e/ojAmZpWCiMPjKyQ4TBr4MYytnTiQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "4.0.14", + "@ai-sdk/provider-utils": "5.0.40", + "@vercel/oidc": "3.2.0" + }, + "engines": { + "node": ">=22" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/mcp": { + "version": "2.0.49", + "resolved": "https://registry.npmjs.org/@ai-sdk/mcp/-/mcp-2.0.49.tgz", + "integrity": "sha512-dD8//65te2C5B4UzPNHGLR7CStR5IvY7o7kXz0YYtPQs+aCdXc7Jb3fuvZcovShfP1KGvPIMTfXhp6Uz+UghJQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "4.0.14", + "@ai-sdk/provider-utils": "5.0.40", + "cross-spawn": "^7.0.6", + "pkce-challenge": "^5.0.1" + }, + "engines": { + "node": ">=22" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/provider": { + "version": "4.0.14", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-4.0.14.tgz", + "integrity": "sha512-yukP2tbcQQErG5gLCMBGvpvb/rM3D3KlTKG6eKKOdNHLHqtNNDEeBxdYrY/JL+O76B2ig5dXY19H/f1HFSvRiQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=22" + } + }, + "node_modules/@ai-sdk/provider-utils": { + "version": "5.0.40", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-5.0.40.tgz", + "integrity": "sha512-zsXPwSAQ9mRJ2hvyITaLOYUyuGrBmzFhJXOg4mllGmla1PfNxZcm4GwqMiV2xQaDgcgBMdUGxXkp9xekZZNIkg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "4.0.14", + "@standard-schema/spec": "^1.1.0", + "@workflow/serde": "4.1.0", + "eventsource-parser": "^3.0.8", + "undici": "^7.29.0" + }, + "engines": { + "node": ">=22" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/react": { + "version": "4.0.102", + "resolved": "https://registry.npmjs.org/@ai-sdk/react/-/react-4.0.102.tgz", + "integrity": "sha512-KOTRsaVUr6QeisktVqm57KfBMqBpxK9f2ay+defGnYsfvXpFE277zg107Q27LSpiTsDpmk2GHMqNWdU9bKaJSw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/mcp": "2.0.49", + "@ai-sdk/provider": "4.0.14", + "@ai-sdk/provider-utils": "5.0.40", + "ai": "7.0.99", + "swr": "^2.4.1", + "throttleit": "2.1.0" + }, + "engines": { + "node": ">=22" + }, + "peerDependencies": { + "react": "^18 || ~19.0.1 || ~19.1.2 || ^19.2.1" + } + }, + "node_modules/@assistant-ui/core": { + "version": "0.3.18", + "resolved": "https://registry.npmjs.org/@assistant-ui/core/-/core-0.3.18.tgz", + "integrity": "sha512-WR5/uqZuI6FNWogIe5EKmP7gdLhQhX38QqaAMAfH9DEPVzw40ON4sZC9vD47rpBj6yrXhHIsbt6jAGUKH4mASQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "assistant-stream": "^0.3.42", + "nanoid": "^6.0.1" + }, + "peerDependencies": { + "@assistant-ui/store": "^0.3.13", + "@assistant-ui/tap": "^0.9.17", + "@types/react": "*", + "assistant-cloud": "^0.2.0", + "react": "^18 || ^19" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "assistant-cloud": { + "optional": true + }, + "react": { + "optional": true + } + } + }, + "node_modules/@assistant-ui/react": { + "version": "0.15.19", + "resolved": "https://registry.npmjs.org/@assistant-ui/react/-/react-0.15.19.tgz", + "integrity": "sha512-+mEXA/ibBSoodj3LFUzRAdYZ+HvlrIg8uexPVjcY3ssksG1zHx8E9TpuwCDxg2h3XrYU8c/UZj/uFlI5tTWMoA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@assistant-ui/core": "^0.3.18", + "@assistant-ui/store": "^0.3.13", + "@assistant-ui/tap": "^0.9.17", + "assistant-cloud": "^0.2.0", + "assistant-stream": "^0.3.42", + "radix-ui": "^1.6.7", + "react-textarea-autosize": "^8.5.9", + "safe-content-frame": "^0.0.30", + "zod": "^4.5.4", + "zustand": "^5.0.15" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^18 || ^19", + "react-dom": "^18 || ^19" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@assistant-ui/store": { + "version": "0.3.13", + "resolved": "https://registry.npmjs.org/@assistant-ui/store/-/store-0.3.13.tgz", + "integrity": "sha512-4u5YAMfjgr+jpJUWZ5f1uqXQuxvIiR8SRg1WWlFbIknk5S1Wli4Jp/nTQsOeXsIw0d7Zdb6FKiQw31gqs5z/Ew==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@assistant-ui/tap": "^0.9.17", + "@types/react": "*", + "react": "^18 || ^19" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "react": { + "optional": true + } + } + }, + "node_modules/@assistant-ui/tap": { + "version": "0.9.17", + "resolved": "https://registry.npmjs.org/@assistant-ui/tap/-/tap-0.9.17.tgz", + "integrity": "sha512-z53TiHiM3ai8XQ5B60Lg8IKCWM0XRf1Sz1jTQtqiuAATzVd6zeFedSHpxq4i5P0zi5uYVdN/dkGEpVTOyPc3ow==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^18 || ^19" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "react": { + "optional": true + } + } + }, + "node_modules/@babel/generator": { + "version": "7.29.8", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.29.8", + "@babel/types": "^7.29.8", + "@jridgewell/gen-mapping": "^0.3.12", + "@jridgewell/trace-mapping": "^0.3.28", + "jsesc": "^3.0.2" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-string-parser": { + "version": "7.29.7", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.29.7", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/parser": { + "version": "7.29.8", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.29.8" + }, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/runtime": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.29.7.tgz", + "integrity": "sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/types": { + "version": "7.29.8", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-string-parser": "^7.29.7", + "@babel/helper-validator-identifier": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@floating-ui/core": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/@floating-ui/core/-/core-1.8.0.tgz", + "integrity": "sha512-0CIZ5itps/8x7BG8dEIhs53BvCUH2PCoogtakwRTut+Arm58sJooJ0AuZhLw2HJYIR5cMLNPBSS728sPho2khQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@floating-ui/utils": "^0.2.12" + } + }, + "node_modules/@floating-ui/dom": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/@floating-ui/dom/-/dom-1.8.0.tgz", + "integrity": "sha512-yXSrzeHZBTZadLOlfyhCkJHNeLJnHRnRInwdZ40L7ZiaAtrBwoYlsDrX3v5zB1Utk7CLfzcOVnVVWoXEky7Ceg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@floating-ui/core": "^1.8.0", + "@floating-ui/utils": "^0.2.12" + } + }, + "node_modules/@floating-ui/react-dom": { + "version": "2.1.9", + "resolved": "https://registry.npmjs.org/@floating-ui/react-dom/-/react-dom-2.1.9.tgz", + "integrity": "sha512-JDjEFGCpImxDCA7JJKviA0M9+RtmJdj0m/NVU5IMgBK+AmZouAQQ7/+2GLH0GXXY0YMw9oXPB8hKdbPYg5QLYg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@floating-ui/dom": "^1.8.0" + }, + "peerDependencies": { + "react": ">=16.8.0", + "react-dom": ">=16.8.0" + } + }, + "node_modules/@floating-ui/utils": { + "version": "0.2.12", + "resolved": "https://registry.npmjs.org/@floating-ui/utils/-/utils-0.2.12.tgz", + "integrity": "sha512-HpCo8tmWzLVad5s2d19EhAz5zqrrQ6s69qd6moPMQvkOuSwDT1YgRfWSVuc4ennqrgv3OHppiOGMQ7oC13yIww==", + "dev": true, + "license": "MIT" + }, + "node_modules/@happy-dom/global-registrator": { + "version": "20.14.5", + "resolved": "https://registry.npmjs.org/@happy-dom/global-registrator/-/global-registrator-20.14.5.tgz", + "integrity": "sha512-B05ID9DhSwLs6mlm1fzlkAtTIvB3duCvjJjfr19LBrlTK7VZtRjDqoTRIVv13GuYuNdhByu8LSZgThwV3Rkj7g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": ">=20.0.0", + "happy-dom": "^20.14.5" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.13", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.0", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.6.0", + "dev": true, + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.31", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@oxc-project/types": { + "version": "0.149.0", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/oxc-project" + } + }, + "node_modules/@quansync/fs": { + "version": "1.0.0", + "dev": true, + "license": "MIT", + "dependencies": { + "quansync": "^1.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sxzz" + } + }, + "node_modules/@radix-ui/number": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@radix-ui/number/-/number-1.1.3.tgz", + "integrity": "sha512-Road2bidD0uu/1BGDOWNdPI06g0lIRy6IF9GZcIrDK2KGItfor8IQwQa+yM2ERgHM1MmHxaxpTzk0/Jp42lNfA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@radix-ui/primitive": { + "version": "1.1.7", + "resolved": "https://registry.npmjs.org/@radix-ui/primitive/-/primitive-1.1.7.tgz", + "integrity": "sha512-rqWnm76nYT8HoNNqEjpgJ7Pw/DrBj5iBTrmEPo6HTX5+VJyBNOqTdv4g89G63HuR5g0AaENoAcH7Is5fF2kZ8Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/@radix-ui/react-accessible-icon": { + "version": "1.1.15", + "resolved": "https://registry.npmjs.org/@radix-ui/react-accessible-icon/-/react-accessible-icon-1.1.15.tgz", + "integrity": "sha512-WTQwcAvQf5sOcuUyi90lKPbhwcvQ+j55cjrSmeaN+L2vKU3DooOvlKw2MDeiJ5IkV5N905KW0/fGojKOBhD11A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-visually-hidden": "1.2.11" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-accordion": { + "version": "1.2.20", + "resolved": "https://registry.npmjs.org/@radix-ui/react-accordion/-/react-accordion-1.2.20.tgz", + "integrity": "sha512-jDhG9FvAEnlhnjrsINbNXcUa4G+L1KqSkJSunkbKEzFRcAb52jvM0PjPxPRvhe1HNc5F5yc0yzzWeeqlH4yBIg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-collapsible": "1.1.20", + "@radix-ui/react-collection": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-controllable-state": "1.2.6" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-alert-dialog": { + "version": "1.1.23", + "resolved": "https://registry.npmjs.org/@radix-ui/react-alert-dialog/-/react-alert-dialog-1.1.23.tgz", + "integrity": "sha512-VAYOiQRqj3GPpYJE0I9J+X8Ip05cyVlNdKOFeiGS2Ou1HHGfpl0BxOyZm6nmVDyU+W+NF3/XLzmjHmVGydhwgA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-dialog": "1.1.23", + "@radix-ui/react-primitive": "2.1.10" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-arrow": { + "version": "1.1.15", + "resolved": "https://registry.npmjs.org/@radix-ui/react-arrow/-/react-arrow-1.1.15.tgz", + "integrity": "sha512-v4zggRcjadnI+ClKDuijlQEW4tw3NoaeHc/PwpKnLoLLKNUG4InLegkstooLcRIUWCs+8L22dGURCVuFfOKfnA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-primitive": "2.1.10" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-aspect-ratio": { + "version": "1.1.15", + "resolved": "https://registry.npmjs.org/@radix-ui/react-aspect-ratio/-/react-aspect-ratio-1.1.15.tgz", + "integrity": "sha512-fy+dyVR+90nelK8rqIznFlxzx7uPcGbhxH8Nfr2bHb4UfSe+e3hklOC0luK0hDwVwnRX7xTRySpsrQVeW+/oNQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-primitive": "2.1.10" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-avatar": { + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/@radix-ui/react-avatar/-/react-avatar-1.2.6.tgz", + "integrity": "sha512-4ULOTJ/mqy2hT9GlWa/MFHxHSvH3nJzHnZM1waNsc5Bonv7i70aNenghXmD97S6OJ81ekXONGGt4nT1r0PfEdA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-callback-ref": "1.1.4", + "@radix-ui/react-use-is-hydrated": "0.1.3", + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-checkbox": { + "version": "1.3.11", + "resolved": "https://registry.npmjs.org/@radix-ui/react-checkbox/-/react-checkbox-1.3.11.tgz", + "integrity": "sha512-Gnptr9pDDQxD3hgq2dtPbtrp/c2qH1mBwIzw3X/ivrMb2e1t0jMTi606fVEqFPaQR1ggXIVQWKj3P2WW9v7zGQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-size": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-collapsible": { + "version": "1.1.20", + "resolved": "https://registry.npmjs.org/@radix-ui/react-collapsible/-/react-collapsible-1.1.20.tgz", + "integrity": "sha512-mcGesGplBnzN2sbvJETzpCNfSMyPnb29q1GRLU+Ib7bJrpIG2ywmRoh2V5VbA2uNvKikKUlVbAPks7JDjz4A8Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-collection": { + "version": "1.1.15", + "resolved": "https://registry.npmjs.org/@radix-ui/react-collection/-/react-collection-1.1.15.tgz", + "integrity": "sha512-9W+B9NPF0NaaPh/1NJd3+KqsnlLqU9H7T2rvww+fp+T/evVXdNAyYcnfRQZFOjkR1ajQp3yORlqnI8soawLvNA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-slot": "1.3.3" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-compose-refs": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@radix-ui/react-compose-refs/-/react-compose-refs-1.1.5.tgz", + "integrity": "sha512-+48PbAAbq3didjJxa+OaWY2ZwgAKsNiRGyeHKszblZMQ+kcpd9pAaT11cMkGEie0vsOi3QdeTE6d5Fe3Gn61kA==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-context": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@radix-ui/react-context/-/react-context-1.2.2.tgz", + "integrity": "sha512-RHCUGwKHDr0hDGg4X7ma4JG4/+12qxw8rkh5QKdDldlCvtja6nUx1Ef/8HVrJze81lEsgLQlqjzjGNHantgnQA==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-context-menu": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/@radix-ui/react-context-menu/-/react-context-menu-2.3.7.tgz", + "integrity": "sha512-CtXP35dxaB5T3zXSd+E3uHe/QpXcpYnZmxp6OaIbfthtfW4wyb77M23BG+bwIJDtsMwEP/YssdsmNyZu7jhWew==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-menu": "2.1.24", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-controllable-state": "1.2.6" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-dialog": { + "version": "1.1.23", + "resolved": "https://registry.npmjs.org/@radix-ui/react-dialog/-/react-dialog-1.1.23.tgz", + "integrity": "sha512-Ksw4WeROkO4rC9k/onilX/Ao2Cr1ku1unMNH+XSCcP4jSXYu7HDsg9n4ojMjVb22XpYjAQ9qfrFlVbru1vXDUA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-dismissable-layer": "1.1.19", + "@radix-ui/react-focus-guards": "1.1.6", + "@radix-ui/react-focus-scope": "1.1.16", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-portal": "1.1.17", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-slot": "1.3.3", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-layout-effect": "1.1.4", + "aria-hidden": "^1.2.4", + "react-remove-scroll": "^2.7.2" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-direction": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-direction/-/react-direction-1.1.4.tgz", + "integrity": "sha512-5pzg4FGQNpExhnhT2zlrP1wZFaYCd1K0nYWoFAdcYoYK868IEigqMX3B3f8yIoRlAhAeDWciLI6ZdCKHF9P4Vg==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-dismissable-layer": { + "version": "1.1.19", + "resolved": "https://registry.npmjs.org/@radix-ui/react-dismissable-layer/-/react-dismissable-layer-1.1.19.tgz", + "integrity": "sha512-8g4pfOL9HoKKLWGiypT+dphVqjFfmcXO5GBnhsG6zI+lxAx/8feQpr+1LSN8Re3hiZ+XkLNS4O9ztK11/LzQ6w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-callback-ref": "1.1.4", + "@radix-ui/react-use-effect-event": "0.0.5" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-dropdown-menu": { + "version": "2.1.24", + "resolved": "https://registry.npmjs.org/@radix-ui/react-dropdown-menu/-/react-dropdown-menu-2.1.24.tgz", + "integrity": "sha512-geq8l2rJkxvkXsT9RMgtUE3P8pITFpTsvYpbySi1IH4fZEABD/Gp85myayFgxk0ktljGMJnCbeFkyTusvSvv7g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-menu": "2.1.24", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-controllable-state": "1.2.6" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-focus-guards": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/@radix-ui/react-focus-guards/-/react-focus-guards-1.1.6.tgz", + "integrity": "sha512-RNOJjfZMTyBM6xYmV3IVGXkPjIhcBAuv48POevAXwrGJhkWZ9p1rFoIS1JFooPuT193AZmRsCPhpoVJxx6OPoQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-focus-scope": { + "version": "1.1.16", + "resolved": "https://registry.npmjs.org/@radix-ui/react-focus-scope/-/react-focus-scope-1.1.16.tgz", + "integrity": "sha512-wmRZ2WWLvmt6KHy2rNPOdPUjwq5xOHY02+m+udwJTn0aNIox/rkskAvJTyTLGhPK6KgrUjlJUJpgmx/+wFiFIQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-callback-ref": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-form": { + "version": "0.1.16", + "resolved": "https://registry.npmjs.org/@radix-ui/react-form/-/react-form-0.1.16.tgz", + "integrity": "sha512-Q4TLEn2A7TAypxwmd6R9EwrlXDvkfYSDMrq9/887AXAGh+G1rH+kYJKSTv+Si9Y0JPKTwKYv6PviAJosysNimA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-label": "2.1.15", + "@radix-ui/react-primitive": "2.1.10" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-hover-card": { + "version": "1.1.23", + "resolved": "https://registry.npmjs.org/@radix-ui/react-hover-card/-/react-hover-card-1.1.23.tgz", + "integrity": "sha512-H8qONfZd3ltrU3+jHCIgITbWo6e1iTKvP9DHdrvYbX48ooRM5FjEDTn16AMwdfuOGkWdZEhpl3PLL/Wk/AnHDQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-dismissable-layer": "1.1.19", + "@radix-ui/react-popper": "1.3.7", + "@radix-ui/react-portal": "1.1.17", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-controllable-state": "1.2.6" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-id": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-id/-/react-id-1.1.4.tgz", + "integrity": "sha512-TMQp2llA+RYn7JcjnrMnz7wN4pcVttPZnRZo52PLQsoLVKzNlVwUeHmfePgTgRluXFvlD3GD5g5MOVVTJCO0qA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-label": { + "version": "2.1.15", + "resolved": "https://registry.npmjs.org/@radix-ui/react-label/-/react-label-2.1.15.tgz", + "integrity": "sha512-o/rdYEwZTTo5tjknnPeyQFU45kUC4i/XyeDPP+HGyi6XqpOP6Zf5Ya5vh/Yfe9Id5JiuWnnAx2XqIeD3UYZt0g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-primitive": "2.1.10" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-menu": { + "version": "2.1.24", + "resolved": "https://registry.npmjs.org/@radix-ui/react-menu/-/react-menu-2.1.24.tgz", + "integrity": "sha512-uW7RVuU6Lp/ZtfeY4b3kL32zccgEWvPv1+cf17ubYzHa9cL8AHokmk36cG/XEiH/smbQvumnieXX9j/e9RqJWA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-collection": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-dismissable-layer": "1.1.19", + "@radix-ui/react-focus-guards": "1.1.6", + "@radix-ui/react-focus-scope": "1.1.16", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-popper": "1.3.7", + "@radix-ui/react-portal": "1.1.17", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-roving-focus": "1.1.19", + "@radix-ui/react-slot": "1.3.3", + "@radix-ui/react-use-callback-ref": "1.1.4", + "aria-hidden": "^1.2.4", + "react-remove-scroll": "^2.7.2" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-menubar": { + "version": "1.1.24", + "resolved": "https://registry.npmjs.org/@radix-ui/react-menubar/-/react-menubar-1.1.24.tgz", + "integrity": "sha512-eeVs0vf7cuqXaM0qLQCPcufImiJNVBXdJDLu7ZGYl2732UH23Qat/foNGrr6vYV3/DdTsBqASoggUFgH14OcZA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-collection": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-menu": "2.1.24", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-roving-focus": "1.1.19", + "@radix-ui/react-use-controllable-state": "1.2.6" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-navigation-menu": { + "version": "1.2.22", + "resolved": "https://registry.npmjs.org/@radix-ui/react-navigation-menu/-/react-navigation-menu-1.2.22.tgz", + "integrity": "sha512-ou7iLEJ+yrhQndkkA4U21XIdS/CS45F4iXIkTZcb6/Ne9EMsOuDudVmCwmDnfFZZ+y1FZqXRNSIgBy+YMvZVZg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-collection": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-dismissable-layer": "1.1.19", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-callback-ref": "1.1.4", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-layout-effect": "1.1.4", + "@radix-ui/react-use-previous": "1.1.4", + "@radix-ui/react-visually-hidden": "1.2.11" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-one-time-password-field": { + "version": "0.1.16", + "resolved": "https://registry.npmjs.org/@radix-ui/react-one-time-password-field/-/react-one-time-password-field-0.1.16.tgz", + "integrity": "sha512-Tj9P6ntAJEw52oq/F0AGknXR4XncxEt7XU47O3xJQOiWfLzEy3d9gtgKfvjSzGxzHkfL+VzvxGu2KTFsloJqXw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/number": "1.1.3", + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-collection": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-roving-focus": "1.1.19", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-effect-event": "0.0.5", + "@radix-ui/react-use-is-hydrated": "0.1.3", + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-password-toggle-field": { + "version": "0.1.11", + "resolved": "https://registry.npmjs.org/@radix-ui/react-password-toggle-field/-/react-password-toggle-field-0.1.11.tgz", + "integrity": "sha512-4gvFnmDXu3dgj21CqsufzIameRvlRd4SBqaWhcrlrNhRo0Y5i/49AmRJYe1fdAM3G2VNBbmin4b0D6cdQocwgw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-effect-event": "0.0.5", + "@radix-ui/react-use-is-hydrated": "0.1.3" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-popover": { + "version": "1.1.23", + "resolved": "https://registry.npmjs.org/@radix-ui/react-popover/-/react-popover-1.1.23.tgz", + "integrity": "sha512-mw58MrBlyHWFisTOYignD0vf/3gdcgAR+9of1s9G/38CbFiUwH1nCDkc0AUM9IrXFgN5Ue8n45j9WCgyM1sbiQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-dismissable-layer": "1.1.19", + "@radix-ui/react-focus-guards": "1.1.6", + "@radix-ui/react-focus-scope": "1.1.16", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-popper": "1.3.7", + "@radix-ui/react-portal": "1.1.17", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-slot": "1.3.3", + "@radix-ui/react-use-controllable-state": "1.2.6", + "aria-hidden": "^1.2.4", + "react-remove-scroll": "^2.7.2" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-popper": { + "version": "1.3.7", + "resolved": "https://registry.npmjs.org/@radix-ui/react-popper/-/react-popper-1.3.7.tgz", + "integrity": "sha512-UsJrrd7w4wuKKTdvd/DNERVlwSlUcyXzjhyDwBk+3aPOsCjOY6ZSbxuw8E6lZTjjfP8Cpd0J8VVkrYUWyGYXyg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@floating-ui/react-dom": "^2.0.0", + "@radix-ui/react-arrow": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-callback-ref": "1.1.4", + "@radix-ui/react-use-layout-effect": "1.1.4", + "@radix-ui/react-use-rect": "1.1.4", + "@radix-ui/react-use-size": "1.1.4", + "@radix-ui/rect": "1.1.3" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-portal": { + "version": "1.1.17", + "resolved": "https://registry.npmjs.org/@radix-ui/react-portal/-/react-portal-1.1.17.tgz", + "integrity": "sha512-vKQLcWypUnwZVvfV7UkGahH2g6ySe8M8R+zYBwPrv5byZ9QAW6cQVvNKo7GgmD+p8aYb6D9JBuvy8/WhOno2wQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-presence": { + "version": "1.1.10", + "resolved": "https://registry.npmjs.org/@radix-ui/react-presence/-/react-presence-1.1.10.tgz", + "integrity": "sha512-3wyzCQ6+ubRA+D4uv9m95JYLXxmOHp05qjrkjeA7uKHHtjpPggQzc6DAb0URl7j67oR0K2foO4ip27TiX037Bw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-primitive": { + "version": "2.1.10", + "resolved": "https://registry.npmjs.org/@radix-ui/react-primitive/-/react-primitive-2.1.10.tgz", + "integrity": "sha512-MucOnzh6hR5mid6VpkbglRAMYMjKLqRnGBbjXkzjK52fuQDd1qbkx78a5P40mkcnVXJdEVxm26E9OPAiUq7nBg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-slot": "1.3.3" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-progress": { + "version": "1.1.16", + "resolved": "https://registry.npmjs.org/@radix-ui/react-progress/-/react-progress-1.1.16.tgz", + "integrity": "sha512-5XnomAsoZZCY+KNTxbIghpGqPruZvKFNlvcAljVAOdDRDsH4/OZQxhtwo5wdtoDM5R6MhJBb2sPnDuRFep3lzg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-primitive": "2.1.10" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-radio-group": { + "version": "1.4.7", + "resolved": "https://registry.npmjs.org/@radix-ui/react-radio-group/-/react-radio-group-1.4.7.tgz", + "integrity": "sha512-cgYFEkntCxppHZgtSZ+7vh0wbZQ+IC7PPMw8DSnRG27B6kDd32/Zw0OJt7dGDigCoprMuWHjg2PvUn3PYvPFoQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-roving-focus": "1.1.19", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-size": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-roving-focus": { + "version": "1.1.19", + "resolved": "https://registry.npmjs.org/@radix-ui/react-roving-focus/-/react-roving-focus-1.1.19.tgz", + "integrity": "sha512-V9jI6hDjT7l3jsCQD9bLNvDLM3tH/gdbOTp7Tefp3hbbgCGQoK7tUvrWiRlcoBHIZ809ElXwNQwVo0B98LuTXQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-collection": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-callback-ref": "1.1.4", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-is-hydrated": "0.1.3", + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-scroll-area": { + "version": "1.2.18", + "resolved": "https://registry.npmjs.org/@radix-ui/react-scroll-area/-/react-scroll-area-1.2.18.tgz", + "integrity": "sha512-Zn5Cd171wxsO3Dfg8HaW6RifTb9CYTKQJHs/G4+LN1GfmJpaQMZQyQxMprVPHpaz7QY4l9BxK2JwQuzHsXC8nA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/number": "1.1.3", + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-callback-ref": "1.1.4", + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-select": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/@radix-ui/react-select/-/react-select-2.3.7.tgz", + "integrity": "sha512-WFGImkmbzcfxeIwq/+4HvRN0pizBwbwQUED4I13ezQsDdfl38ZntN6TmR8XaSzPBqoCToe8rF75j6NPNDSzhbg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/number": "1.1.3", + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-collection": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-dismissable-layer": "1.1.19", + "@radix-ui/react-focus-guards": "1.1.6", + "@radix-ui/react-focus-scope": "1.1.16", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-popper": "1.3.7", + "@radix-ui/react-portal": "1.1.17", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-slot": "1.3.3", + "@radix-ui/react-use-callback-ref": "1.1.4", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-layout-effect": "1.1.4", + "@radix-ui/react-use-previous": "1.1.4", + "@radix-ui/react-visually-hidden": "1.2.11", + "aria-hidden": "^1.2.4", + "react-remove-scroll": "^2.7.2" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-separator": { + "version": "1.1.15", + "resolved": "https://registry.npmjs.org/@radix-ui/react-separator/-/react-separator-1.1.15.tgz", + "integrity": "sha512-jOLO4lssEzWpoDu7G+Ze4VjwMRUBt291pnZD0gmalREZipnTX3wadQo7Fy48GCTfe14/YRN6rw/rOJqrE85Wxw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-primitive": "2.1.10" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-slider": { + "version": "1.4.7", + "resolved": "https://registry.npmjs.org/@radix-ui/react-slider/-/react-slider-1.4.7.tgz", + "integrity": "sha512-mTSLf1GC/C0moWjTbvCM6Qn/gBjvlFt1azuWF2v7MN5C3Zq2U2J2lN3ZEYkpujuOU5Ro7A28wkviSxaKnG0BYg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/number": "1.1.3", + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-collection": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-layout-effect": "1.1.4", + "@radix-ui/react-use-previous": "1.1.4", + "@radix-ui/react-use-size": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-slot": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@radix-ui/react-slot/-/react-slot-1.3.3.tgz", + "integrity": "sha512-qx7oqnYbxnK9kYI9m317qmFmEgo6ywqWvbTogdj7cL9p3/yx4M48p7Rnw5z3H890cL/ow/EeWJsuTykeZVXP5Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-compose-refs": "1.1.5" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-switch": { + "version": "1.3.7", + "resolved": "https://registry.npmjs.org/@radix-ui/react-switch/-/react-switch-1.3.7.tgz", + "integrity": "sha512-48tB/4dn2UVLBCYhTu9AuR63IHl73l/qLbLgxd86noTUor4/K4LFDAcYjK+isP5313qxaFpjPVogE7+Y0/V3Kw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-size": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-tabs": { + "version": "1.1.21", + "resolved": "https://registry.npmjs.org/@radix-ui/react-tabs/-/react-tabs-1.1.21.tgz", + "integrity": "sha512-UKxJlZid7FVtsk/WTxj4i4uSEgj2Au+KBbS7SQyTlzMhhn+86Cz3tISZdTa87bfEfcuvZezf2ZsxD4xuEKtkog==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-roving-focus": "1.1.19", + "@radix-ui/react-use-controllable-state": "1.2.6" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-toast": { + "version": "1.2.23", + "resolved": "https://registry.npmjs.org/@radix-ui/react-toast/-/react-toast-1.2.23.tgz", + "integrity": "sha512-ofhyAsYaocRGOs/n0XWdUOSVzEAG6BfrMVM8z0c0kLEWY38w/0WuMFPTJP/HVaZPYkMvHZoKIIhNcjbTCBILPg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-collection": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-dismissable-layer": "1.1.19", + "@radix-ui/react-portal": "1.1.17", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-callback-ref": "1.1.4", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-layout-effect": "1.1.4", + "@radix-ui/react-visually-hidden": "1.2.11" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-toggle": { + "version": "1.1.18", + "resolved": "https://registry.npmjs.org/@radix-ui/react-toggle/-/react-toggle-1.1.18.tgz", + "integrity": "sha512-7lonPlKfSacd20GlOBx2ltuVKz9oqWYZz+oMQyOltw6t1y2nyftj2ZmwwUHYn49kqfDWcp8dNZm5NgV+5Z+mug==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-controllable-state": "1.2.6" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-toggle-group": { + "version": "1.1.19", + "resolved": "https://registry.npmjs.org/@radix-ui/react-toggle-group/-/react-toggle-group-1.1.19.tgz", + "integrity": "sha512-OtnwuSVjd1Ofi+AdnvhsjQdyuhCDwYs1w9RyB5BN/OavXOVQo42SYqQjwUnbPnaiPFBpQ9aX70dWeee+v2oBLA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-roving-focus": "1.1.19", + "@radix-ui/react-toggle": "1.1.18", + "@radix-ui/react-use-controllable-state": "1.2.6" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-toolbar": { + "version": "1.1.19", + "resolved": "https://registry.npmjs.org/@radix-ui/react-toolbar/-/react-toolbar-1.1.19.tgz", + "integrity": "sha512-Ph0IvtYw4VB12ZnZg+YtrGs8yJQsnizwo/zu0R4Y/nWugtJzA7Pg1eWeuDR9+LSqn+xjamss+UOSOJJJ4gx8jw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-roving-focus": "1.1.19", + "@radix-ui/react-separator": "1.1.15", + "@radix-ui/react-toggle-group": "1.1.19" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-tooltip": { + "version": "1.2.16", + "resolved": "https://registry.npmjs.org/@radix-ui/react-tooltip/-/react-tooltip-1.2.16.tgz", + "integrity": "sha512-6EamKFRRnlpdadndbZ6LMwycfwkwPte1B42hs6QA0gYhjaOKqW4PZ4pjaW9UrlDX5eVt/OjncE7BFTPL5nmZhg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-dismissable-layer": "1.1.19", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-popper": "1.3.7", + "@radix-ui/react-portal": "1.1.17", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-slot": "1.3.3", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-layout-effect": "1.1.4", + "@radix-ui/react-visually-hidden": "1.2.11" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-callback-ref": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-callback-ref/-/react-use-callback-ref-1.1.4.tgz", + "integrity": "sha512-R6OUY2e2fA6Yn6s+VSx5KBV6Nx8LQEhu+cz7LCej18rQ1HLyg9PSC9jP/ZNx0o6FAIK9c0F1kHylzSxKsdlkrQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-controllable-state": { + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-controllable-state/-/react-use-controllable-state-1.2.6.tgz", + "integrity": "sha512-uEQJGT97ZA/TgP/Hydw47lHu+/vQj6z/0jA+WeTbK1o9Rx45GImjpD0tc3W5ad3D6XTSR6e1yEO0FvGq6WQfVQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-use-effect-event": "0.0.5", + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-effect-event": { + "version": "0.0.5", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-effect-event/-/react-use-effect-event-0.0.5.tgz", + "integrity": "sha512-7cshFL8HGS/7HEiHH+9kL9HBwp2sa9yX18Knwek6KYWmXwM7pegMgta2AXMQKI+rq3JnfSj9x8wYqFMTdG1Jgg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-escape-keydown": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-escape-keydown/-/react-use-escape-keydown-1.1.5.tgz", + "integrity": "sha512-ge3ipobwSXTj4JyVtswQ7qZj0ZHdtbGuOno/LrgAAeSxtsJ6Vs4Gz5IkPH2bmqpjcLUFoqGhA/mueuIf63UXlA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-use-callback-ref": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-is-hydrated": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-is-hydrated/-/react-use-is-hydrated-0.1.3.tgz", + "integrity": "sha512-umO/aJ+82CpOnhDZUTbILCQf7kU/g0iv+oGs/Q8jw7IkhWBzaEP4sA268PhFAJTFetbwp3ICc6ktpI4TqtxcIw==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-layout-effect": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-layout-effect/-/react-use-layout-effect-1.1.4.tgz", + "integrity": "sha512-K20DkRkUwDnxEYMBPcg3Y6voLkEy5p5QQmszZgLngKKiC7dzBR/aEuK3w1qlx2JWDUNH6FluahYdgR3BP+QbYw==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-previous": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-previous/-/react-use-previous-1.1.4.tgz", + "integrity": "sha512-XoSLhbRbqxFtgJoi2fNHA3C6pDlY34x508vUpUGoFZfvePfHXHbE1lC4FYFMnJWgiCRroSTw6fOsXQoVS9RwZg==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-rect": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-rect/-/react-use-rect-1.1.4.tgz", + "integrity": "sha512-cSOCh6JlkmfjLyNcLiu2nB4v+nm+dkZ+Q5KHWk/soo4U7ZLiEQFKHK9/YmtBHjfCEaU43IBKQOc4/uJmCaiCTQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/rect": "1.1.3" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-size": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-size/-/react-use-size-1.1.4.tgz", + "integrity": "sha512-D3anSY15EJoxrihpsXI6SMrmmonnQtR2ni7arO+Lfdg3O95b9hNXxONk8jA5C8ANdF/h5HMAxejgs8PWJ6rlhw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-visually-hidden": { + "version": "1.2.11", + "resolved": "https://registry.npmjs.org/@radix-ui/react-visually-hidden/-/react-visually-hidden-1.2.11.tgz", + "integrity": "sha512-NFS86RYYZb4/exihaESBGOpMJFz8MGLAfu3mOBSGByVnVPC9JPASfYubxd/8KbkQK0sYAv8lVQDEQukDX/qXvQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-primitive": "2.1.10" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/rect": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@radix-ui/rect/-/rect-1.1.3.tgz", + "integrity": "sha512-JtyZR+mqgBibTo8xea3B6ZRmzZiM/YeVBtUkas6zMuXjAlfIFIW2FgqeM9eLyvEaYX66vr6DJMK+4U6LV0KhNw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@rolldown/binding-android-arm-eabi": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm-eabi/-/binding-android-arm-eabi-1.2.8.tgz", + "integrity": "sha512-tN5aztYkKCte4i5SIrrz5yK/HMjEuCqCSCJa418jOV8tZ1cBY3YF2otxB1ktPxzsLA1BeTqwapK0bfjxNvHJVw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-android-arm64": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.2.8.tgz", + "integrity": "sha512-dIYTWl9XprMUiQFoc55KUyk/oS8SKYH3zFl0LTR7RT0Xj4hgSVyuJcroH8JUu8RcpF8fTB6E0aOwCkZoYPcDSQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-arm64": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.2.8.tgz", + "integrity": "sha512-PCSDQGXD2IyTEFrcgPyBM8jJuGmrbCMuoIOXdbEGVemruKACXoLQJrb+A45Z0L5t1RQkdfJprAYPkikbh7dzdA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-x64": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.2.8.tgz", + "integrity": "sha512-Uk7lRsGhPFHVX/sAUC6D5H9Ol30dFHd6iquokll2th3LpdJ3F5CzQB+7DHn0Ri2mG+U7k2zXiPHDrwZenXhwSA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-freebsd-x64": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.2.8.tgz", + "integrity": "sha512-DjszaTEVogPqA5bYzsEeqDCQxbcp2fexQwKcRspYji2yzR68fCf+e4fx6kBSRDwX5/brZaHw/hWS9+A/+/w9sQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm-gnueabihf": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.2.8.tgz", + "integrity": "sha512-zmwa7FTmdzB6aaEEuuls18H6Ap5JmJPSoPTuXixeJZV6tG40SyLkApQtz1g8ptZtiEKqj9OM0oNLPh1AgvE31Q==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-gnu": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.2.8.tgz", + "integrity": "sha512-KdYQDPHwJVnbFwdTGMgxsI9SqblBlz6STGM+w1We/d5B8OWWidYH0MwkU/uA1wM5fIpO2MkOVxXrNzzuZhw9ew==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-musl": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.2.8.tgz", + "integrity": "sha512-jFJTifHnNPY+yzOoNZQfSIysrVyXzEQPhPnOUjmD1bcQGHH6s7c8cViKWar8YplQImE5N9JRqMCLrM2CdxOrZA==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-ppc64-gnu": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.2.8.tgz", + "integrity": "sha512-FhiOziBDWPBjbcmRzfLyIJnaP7AVMFXT7YCXPjXxj7wKU3vx24RjrCNN/zjvVa+N2vVoHJwCoUBvsrN/DG3zIA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-s390x-gnu": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.2.8.tgz", + "integrity": "sha512-WnHfADMzOV2Y55wlx1hzzQnar/wDt/VdvWSD99r18Mz9ylNieIGOkRx3UV21h7m/eJvjySYJkO26VvGNFkwsIQ==", + "cpu": [ + "s390x" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-gnu": { + "version": "1.2.8", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-musl": { + "version": "1.2.8", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-openharmony-arm64": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.2.8.tgz", + "integrity": "sha512-637Ke4kWSy6rp9cxQ9gMOXlxPgIw/c1beASV4M//3+9I4uwBVOOl74G+e3zyU3u19U7RkRl/HuewixZ/Z6+Rjg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-win32-arm64-msvc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.2.8.tgz", + "integrity": "sha512-xWBkPOF1Q9k/Gv1nQXnVdLxKu74jXppuOM4Z3mnypVUJJJwLsMl7hNJGRAUJoG8A5MgOI1ACKM+wBFxSJzKy4A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-win32-x64-msvc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.2.8.tgz", + "integrity": "sha512-uz2ZvfgXbxqNwijjjbxrnvALwpyODDcgc1T1N8N3rf/DXKQmaFwmB4LX4yyjggpwN2obdQLb2rgirX5ffCWYng==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/pluginutils": { + "version": "1.0.1", + "dev": true, + "license": "MIT" + }, + "node_modules/@standard-schema/spec": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", + "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/bun": { + "version": "1.4.2", + "dev": true, + "license": "MIT", + "dependencies": { + "bun-types": "1.4.2" + } + }, + "node_modules/@types/node": { + "version": "26.5.1", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~8.9.0" + } + }, + "node_modules/@types/react": { + "version": "19.3.0", + "dev": true, + "license": "MIT", + "dependencies": { + "csstype": "^3.2.2" + } + }, + "node_modules/@types/react-dom": { + "version": "19.3.0", + "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-19.3.0.tgz", + "integrity": "sha512-ZI7bU42mZXXKHn/qNLEw2IrbiINU7X5+vfgdixBHkCNpYWXjKgfQ/P+uyGb5CjOLB9UcnTeg3rylQtV2hym44Q==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "^19.3.0" + } + }, + "node_modules/@types/whatwg-mimetype": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@types/whatwg-mimetype/-/whatwg-mimetype-3.0.2.tgz", + "integrity": "sha512-c2AKvDT8ToxLIOUlN51gTiHXflsfIFisS4pO7pDPoKouJCESkhZnEy623gwP9laCy5lnLDAw1vAzu2vM2YLOrA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/ws": { + "version": "8.18.1", + "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.18.1.tgz", + "integrity": "sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@vercel/oidc": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/@vercel/oidc/-/oidc-3.2.0.tgz", + "integrity": "sha512-UycprH3T6n3jH0k44NHMa7pnFHGu/N05MjojYr+Mc6I7obkoLIJujSWwin1pCvdy/eOxrI/l3uDLQsmcrOb4ug==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">= 20" + } + }, + "node_modules/@workflow/serde": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@workflow/serde/-/serde-4.1.0.tgz", + "integrity": "sha512-pav4F2BoirECWR7Nf1TKt+2eETcBj7jj4cBefQ8VXQCA6NPkaKeLfj/zMgi+3zYV5ZIBT4GuUiphsj0/b9hPQQ==", + "dev": true, + "license": "Apache-2.0" + }, + "node_modules/ai": { + "version": "7.0.99", + "resolved": "https://registry.npmjs.org/ai/-/ai-7.0.99.tgz", + "integrity": "sha512-Ov+3j/nSajaVH5hO8C94wN9wisG5tAJ8HWsLV9d/+nlzrRGUh3oYO3Kp1fRyUNWjAWLSpGLHLPaHCdfxb307Vg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/gateway": "4.0.80", + "@ai-sdk/provider": "4.0.14", + "@ai-sdk/provider-utils": "5.0.40" + }, + "engines": { + "node": ">=22" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/ansis": { + "version": "4.4.0", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=14" + } + }, + "node_modules/aria-hidden": { + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/aria-hidden/-/aria-hidden-1.2.6.tgz", + "integrity": "sha512-ik3ZgC9dY/lYVVM++OISsaYDeg1tb0VtP5uL3ouh1koGOaUMDPpbFIei4JkFimWUFPn90sbMNMXQAIVOlnYKJA==", + "dev": true, + "license": "MIT", + "dependencies": { + "tslib": "^2.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/assistant-cloud": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/assistant-cloud/-/assistant-cloud-0.2.0.tgz", + "integrity": "sha512-LMvPaufIfZ0dpByMsNeuPFY6UeKQ1oiSBktUJuhjL/2eTZLSzzjIWumiLALlpdvUZbbJCRHy8j0ZgzlZJJlEfQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "assistant-stream": "^0.3.42" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.9.0", + "@opentelemetry/exporter-trace-otlp-http": ">=0.200.0", + "@opentelemetry/sdk-trace-base": "^2.1.0" + }, + "peerDependenciesMeta": { + "@opentelemetry/api": { + "optional": true + }, + "@opentelemetry/exporter-trace-otlp-http": { + "optional": true + }, + "@opentelemetry/sdk-trace-base": { + "optional": true + } + } + }, + "node_modules/assistant-stream": { + "version": "0.3.42", + "resolved": "https://registry.npmjs.org/assistant-stream/-/assistant-stream-0.3.42.tgz", + "integrity": "sha512-5dQxc7XX92LuJ8wuHaOi/vkHitz2DARABzoE364O7o4jiePWalDpDpqU9bN+bXTvnSiRrpesy6t3Z3oDqRkdQw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@standard-schema/spec": "^1.1.0", + "nanoid": "^6.0.1", + "secure-json-parse": "^4.1.0" + }, + "peerDependencies": { + "ioredis": "^5.10.1 || ^6.0.0", + "redis": "^5.12.1" + }, + "peerDependenciesMeta": { + "ioredis": { + "optional": true + }, + "redis": { + "optional": true + } + } + }, + "node_modules/ast-kit": { + "version": "2.2.0", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.28.5", + "pathe": "^2.0.3" + }, + "engines": { + "node": ">=20.19.0" + }, + "funding": { + "url": "https://github.com/sponsors/sxzz" + } + }, + "node_modules/birpc": { + "version": "2.9.0", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/buffer-image-size": { + "version": "0.6.4", + "resolved": "https://registry.npmjs.org/buffer-image-size/-/buffer-image-size-0.6.4.tgz", + "integrity": "sha512-nEh+kZOPY1w+gcCMobZ6ETUp9WfibndnosbpwB1iJk/8Gt5ZF2bhS6+B6bPYz424KtwsR6Rflc3tCz1/ghX2dQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + }, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/bun-types": { + "version": "1.4.2", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/cac": { + "version": "6.7.14", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/chokidar": { + "version": "4.0.3", + "dev": true, + "license": "MIT", + "dependencies": { + "readdirp": "^4.0.1" + }, + "engines": { + "node": ">= 14.16.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "dev": true, + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/csstype": { + "version": "3.2.3", + "dev": true, + "license": "MIT" + }, + "node_modules/debug": { + "version": "4.4.3", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/defu": { + "version": "6.1.7", + "dev": true, + "license": "MIT" + }, + "node_modules/dequal": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", + "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/detect-node-es": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/detect-node-es/-/detect-node-es-1.1.0.tgz", + "integrity": "sha512-ypdmJU/TbBby2Dxibuv7ZLW3Bs1QEmM7nHjEANfohJLvE0XVujisn1qPJcZxg+qDucsr+bP6fLD1rPS3AhJ7EQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/diff": { + "version": "8.0.4", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.3.1" + } + }, + "node_modules/dts-resolver": { + "version": "2.1.3", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=20.19.0" + }, + "funding": { + "url": "https://github.com/sponsors/sxzz" + }, + "peerDependencies": { + "oxc-resolver": ">=11.0.0" + }, + "peerDependenciesMeta": { + "oxc-resolver": { + "optional": true + } + } + }, + "node_modules/empathic": { + "version": "2.0.1", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14" + } + }, + "node_modules/entities": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/entities/-/entities-7.0.1.tgz", + "integrity": "sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/eventsource-parser": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.1.1.tgz", + "integrity": "sha512-EKN1vKAMcZ8MlYMpaNuxN6R9yakzH6uajHcHVTqWJzvu5pWw9DyhbP35HH8MVBQ+dZjAfDxk+A8NiR9KWaXiyQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/fdir": { + "version": "6.5.0", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/get-nonce": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-nonce/-/get-nonce-1.0.1.tgz", + "integrity": "sha512-FJhYRoDaiatfEkUK8HKlicmu/3SGFD51q3itKDGoSTysQJBnfOcxU5GxnhE1E6soB76MbT0MBtnKJuXyAx+96Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/get-tsconfig": { + "version": "4.14.3", + "dev": true, + "license": "MIT", + "dependencies": { + "resolve-pkg-maps": "^1.0.0" + }, + "funding": { + "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" + } + }, + "node_modules/happy-dom": { + "version": "20.14.5", + "resolved": "https://registry.npmjs.org/happy-dom/-/happy-dom-20.14.5.tgz", + "integrity": "sha512-x/RzkpWO40bTjIoT30iQtt64FLLmH/iRcUCN2X//bLx7H3ifkdfPXyqsro/OYtqzIAhiLMMA7mmiOR9C3NOKjQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": ">=20.0.0", + "@types/whatwg-mimetype": "^3.0.2", + "@types/ws": "^8.18.1", + "buffer-image-size": "^0.6.4", + "entities": "^7.0.1", + "whatwg-mimetype": "^3.0.0", + "ws": "^8.21.0" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/hookable": { + "version": "5.5.3", + "dev": true, + "license": "MIT" + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true, + "license": "ISC" + }, + "node_modules/jiti": { + "version": "2.7.0", + "dev": true, + "license": "MIT", + "bin": { + "jiti": "lib/jiti-cli.mjs" + } + }, + "node_modules/jsesc": { + "version": "3.1.0", + "dev": true, + "license": "MIT", + "bin": { + "jsesc": "bin/jsesc" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/json-schema": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz", + "integrity": "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==", + "dev": true, + "license": "(AFL-2.1 OR BSD-3-Clause)" + }, + "node_modules/ms": { + "version": "2.1.3", + "dev": true, + "license": "MIT" + }, + "node_modules/nanoid": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-6.0.1.tgz", + "integrity": "sha512-3wVS3i51pE2pi1k5FFL/95BGfVS0kSsvDVuGXHOtxox/TywUmtgq+3qiTOTbs9J7KfHaXPiN171k/A6dBnaXFw==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.js" + }, + "engines": { + "node": "^22 || ^24 || >=26" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/pathe": { + "version": "2.0.3", + "dev": true, + "license": "MIT" + }, + "node_modules/picomatch": { + "version": "4.0.7", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/pkce-challenge": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/pkce-challenge/-/pkce-challenge-5.0.1.tgz", + "integrity": "sha512-wQ0b/W4Fr01qtpHlqSqspcj3EhBvimsdh0KlHhH8HRZnMsEa0ea2fTULOXOS9ccQr3om+GcGRk4e+isrZWV8qQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/quansync": { + "version": "1.0.0", + "dev": true, + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/antfu" + }, + { + "type": "individual", + "url": "https://github.com/sponsors/sxzz" + } + ], + "license": "MIT" + }, + "node_modules/radix-ui": { + "version": "1.6.7", + "resolved": "https://registry.npmjs.org/radix-ui/-/radix-ui-1.6.7.tgz", + "integrity": "sha512-QBdhh1arIEUvPC0dQ5+nwWAxt7+N+oP/9jPwjJkGFoSk/sqxg32gJtSXGtFh8frAIcS6oC9cx2Q+7KYCQLOAeA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-accessible-icon": "1.1.15", + "@radix-ui/react-accordion": "1.2.20", + "@radix-ui/react-alert-dialog": "1.1.23", + "@radix-ui/react-arrow": "1.1.15", + "@radix-ui/react-aspect-ratio": "1.1.15", + "@radix-ui/react-avatar": "1.2.6", + "@radix-ui/react-checkbox": "1.3.11", + "@radix-ui/react-collapsible": "1.1.20", + "@radix-ui/react-collection": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-context-menu": "2.3.7", + "@radix-ui/react-dialog": "1.1.23", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-dismissable-layer": "1.1.19", + "@radix-ui/react-dropdown-menu": "2.1.24", + "@radix-ui/react-focus-guards": "1.1.6", + "@radix-ui/react-focus-scope": "1.1.16", + "@radix-ui/react-form": "0.1.16", + "@radix-ui/react-hover-card": "1.1.23", + "@radix-ui/react-label": "2.1.15", + "@radix-ui/react-menu": "2.1.24", + "@radix-ui/react-menubar": "1.1.24", + "@radix-ui/react-navigation-menu": "1.2.22", + "@radix-ui/react-one-time-password-field": "0.1.16", + "@radix-ui/react-password-toggle-field": "0.1.11", + "@radix-ui/react-popover": "1.1.23", + "@radix-ui/react-popper": "1.3.7", + "@radix-ui/react-portal": "1.1.17", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-progress": "1.1.16", + "@radix-ui/react-radio-group": "1.4.7", + "@radix-ui/react-roving-focus": "1.1.19", + "@radix-ui/react-scroll-area": "1.2.18", + "@radix-ui/react-select": "2.3.7", + "@radix-ui/react-separator": "1.1.15", + "@radix-ui/react-slider": "1.4.7", + "@radix-ui/react-slot": "1.3.3", + "@radix-ui/react-switch": "1.3.7", + "@radix-ui/react-tabs": "1.1.21", + "@radix-ui/react-toast": "1.2.23", + "@radix-ui/react-toggle": "1.1.18", + "@radix-ui/react-toggle-group": "1.1.19", + "@radix-ui/react-toolbar": "1.1.19", + "@radix-ui/react-tooltip": "1.2.16", + "@radix-ui/react-use-callback-ref": "1.1.4", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-effect-event": "0.0.5", + "@radix-ui/react-use-escape-keydown": "1.1.5", + "@radix-ui/react-use-is-hydrated": "0.1.3", + "@radix-ui/react-use-layout-effect": "1.1.4", + "@radix-ui/react-use-size": "1.1.4", + "@radix-ui/react-visually-hidden": "1.2.11" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/react": { + "version": "19.3.0", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/react-dom": { + "version": "19.3.0", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.3.0.tgz", + "integrity": "sha512-JDk8dgif51OjFoDE70+OT9ICyYr+69HlmihNwp1+Nsfbna3t5sIiCa9ZJktDmQ4/1b/rn26hIAR2uYXDMr5r0Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "scheduler": "^0.28.0" + }, + "peerDependencies": { + "react": "^19.3.0" + } + }, + "node_modules/react-remove-scroll": { + "version": "2.7.2", + "resolved": "https://registry.npmjs.org/react-remove-scroll/-/react-remove-scroll-2.7.2.tgz", + "integrity": "sha512-Iqb9NjCCTt6Hf+vOdNIZGdTiH1QSqr27H/Ek9sv/a97gfueI/5h1s3yRi1nngzMUaOOToin5dI1dXKdXiF+u0Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "react-remove-scroll-bar": "^2.3.7", + "react-style-singleton": "^2.2.3", + "tslib": "^2.1.0", + "use-callback-ref": "^1.3.3", + "use-sidecar": "^1.1.3" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/react-remove-scroll-bar": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/react-remove-scroll-bar/-/react-remove-scroll-bar-2.3.8.tgz", + "integrity": "sha512-9r+yi9+mgU33AKcj6IbT9oRCO78WriSj6t/cF8DWBZJ9aOGPOTEDvdUDz1FwKim7QXWwmHqtdHnRJfhAxEG46Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "react-style-singleton": "^2.2.2", + "tslib": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/react-style-singleton": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/react-style-singleton/-/react-style-singleton-2.2.3.tgz", + "integrity": "sha512-b6jSvxvVnyptAiLjbkWLE/lOnR4lfTtDAl+eUC7RZy+QQWc6wRzIV2CE6xBuMmDxc2qIihtDCZD5NPOFl7fRBQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "get-nonce": "^1.0.0", + "tslib": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/react-textarea-autosize": { + "version": "8.5.9", + "resolved": "https://registry.npmjs.org/react-textarea-autosize/-/react-textarea-autosize-8.5.9.tgz", + "integrity": "sha512-U1DGlIQN5AwgjTyOEnI1oCcMuEr1pv1qOtklB2l4nyMGbHzWrI0eFsYK0zos2YWqAolJyG0IWJaqWmWj5ETh0A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.20.13", + "use-composed-ref": "^1.3.0", + "use-latest": "^1.2.1" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, + "node_modules/readdirp": { + "version": "4.1.2", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 14.18.0" + }, + "funding": { + "type": "individual", + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/resolve-pkg-maps": { + "version": "1.0.0", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1" + } + }, + "node_modules/rolldown": { + "version": "1.2.8", + "dev": true, + "license": "MIT", + "dependencies": { + "@oxc-project/types": "=0.149.0", + "@rolldown/pluginutils": "^1.0.0" + }, + "bin": { + "rolldown": "bin/cli.mjs" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "optionalDependencies": { + "@rolldown/binding-android-arm-eabi": "1.2.8", + "@rolldown/binding-android-arm64": "1.2.8", + "@rolldown/binding-darwin-arm64": "1.2.8", + "@rolldown/binding-darwin-x64": "1.2.8", + "@rolldown/binding-freebsd-x64": "1.2.8", + "@rolldown/binding-linux-arm-gnueabihf": "1.2.8", + "@rolldown/binding-linux-arm64-gnu": "1.2.8", + "@rolldown/binding-linux-arm64-musl": "1.2.8", + "@rolldown/binding-linux-ppc64-gnu": "1.2.8", + "@rolldown/binding-linux-s390x-gnu": "1.2.8", + "@rolldown/binding-linux-x64-gnu": "1.2.8", + "@rolldown/binding-linux-x64-musl": "1.2.8", + "@rolldown/binding-openharmony-arm64": "1.2.8", + "@rolldown/binding-win32-arm64-msvc": "1.2.8", + "@rolldown/binding-win32-x64-msvc": "1.2.8" + } + }, + "node_modules/rolldown-plugin-dts": { + "version": "0.13.14", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/generator": "^7.28.0", + "@babel/parser": "^7.28.0", + "@babel/types": "^7.28.1", + "ast-kit": "^2.1.1", + "birpc": "^2.5.0", + "debug": "^4.4.1", + "dts-resolver": "^2.1.1", + "get-tsconfig": "^4.10.1" + }, + "engines": { + "node": ">=20.18.0" + }, + "funding": { + "url": "https://github.com/sponsors/sxzz" + }, + "peerDependencies": { + "@typescript/native-preview": ">=7.0.0-dev.20250601.1", + "rolldown": "^1.0.0-beta.9", + "typescript": "^5.0.0", + "vue-tsc": "^2.2.0 || ^3.0.0" + }, + "peerDependenciesMeta": { + "@typescript/native-preview": { + "optional": true + }, + "typescript": { + "optional": true + }, + "vue-tsc": { + "optional": true + } + } + }, + "node_modules/safe-content-frame": { + "version": "0.0.30", + "resolved": "https://registry.npmjs.org/safe-content-frame/-/safe-content-frame-0.0.30.tgz", + "integrity": "sha512-t8XO/59b+YaJCmCpJ3aTZ8TyB6LzG7VzCtUO3u8h0jUsK/wCvQPMro8ioCA6AQr0Ve8Cv9zunFp1VcFGm1aybg==", + "dev": true, + "license": "MIT" + }, + "node_modules/scheduler": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.28.0.tgz", + "integrity": "sha512-juorfCmIkIw8tT+p5BXSm6PJjQF/ycEYmKyzURCIt/RaZIhL+PulbQ9Yu2z1HdOJDdqDTlxA1+xKBmHXJsczAw==", + "dev": true, + "license": "MIT" + }, + "node_modules/secure-json-parse": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-4.1.0.tgz", + "integrity": "sha512-l4KnYfEyqYJxDwlNVyRfO2E4NTHfMKAWdUuA8J0yve2Dz/E/PdBepY03RvyJpssIpRFwJoCD55wA+mEDs6ByWA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/semver": { + "version": "7.8.5", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/swr": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/swr/-/swr-2.5.1.tgz", + "integrity": "sha512-BRw55e8r0B7SpDN20CAzoQAHl7y1yP7/Zt7oqUjMv0vSt2u2Xnkm88Ws+VypbV9BXHQVuSuyVq7zMjO16wSExw==", + "dev": true, + "license": "MIT", + "dependencies": { + "dequal": "^2.0.3", + "use-sync-external-store": "^1.6.0" + }, + "peerDependencies": { + "react": "^16.11.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, + "node_modules/throttleit": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/throttleit/-/throttleit-2.1.0.tgz", + "integrity": "sha512-nt6AMGKW1p/70DF/hGBdJB57B8Tspmbp5gfJ8ilhLnt7kkr2ye7hzD6NVG8GGErk2HWF34igrL2CXmNIkzKqKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/tinyexec": { + "version": "1.3.1", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/tinyglobby": { + "version": "0.2.17", + "dev": true, + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.4" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/tsdown": { + "version": "0.12.9", + "dev": true, + "license": "MIT", + "dependencies": { + "ansis": "^4.1.0", + "cac": "^6.7.14", + "chokidar": "^4.0.3", + "debug": "^4.4.1", + "diff": "^8.0.2", + "empathic": "^2.0.0", + "hookable": "^5.5.3", + "rolldown": "^1.0.0-beta.19", + "rolldown-plugin-dts": "^0.13.12", + "semver": "^7.7.2", + "tinyexec": "^1.0.1", + "tinyglobby": "^0.2.14", + "unconfig": "^7.3.2" + }, + "bin": { + "tsdown": "dist/run.mjs" + }, + "engines": { + "node": ">=18.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sxzz" + }, + "peerDependencies": { + "@arethetypeswrong/core": "^0.18.1", + "publint": "^0.3.0", + "typescript": "^5.0.0", + "unplugin-lightningcss": "^0.4.0", + "unplugin-unused": "^0.5.0" + }, + "peerDependenciesMeta": { + "@arethetypeswrong/core": { + "optional": true + }, + "publint": { + "optional": true + }, + "typescript": { + "optional": true + }, + "unplugin-lightningcss": { + "optional": true + }, + "unplugin-unused": { + "optional": true + } + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "dev": true, + "license": "0BSD" + }, + "node_modules/typescript": { + "version": "5.9.3", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/unconfig": { + "version": "7.5.0", + "dev": true, + "license": "MIT", + "dependencies": { + "@quansync/fs": "^1.0.0", + "defu": "^6.1.4", + "jiti": "^2.6.1", + "quansync": "^1.0.0", + "unconfig-core": "7.5.0" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/unconfig-core": { + "version": "7.5.0", + "dev": true, + "license": "MIT", + "dependencies": { + "@quansync/fs": "^1.0.0", + "quansync": "^1.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/undici": { + "version": "7.29.1", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.29.1.tgz", + "integrity": "sha512-RYONW2MeafgYlkVOKYKkA/Ag7BmXqgIWCa8t1m0JcxrQg9pI9lEqRhAOruOBCbAohOa/gkCF+iPi9hrgvTzu6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=20.18.1" + } + }, + "node_modules/undici-types": { + "version": "8.9.0", + "dev": true, + "license": "MIT" + }, + "node_modules/use-callback-ref": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/use-callback-ref/-/use-callback-ref-1.3.3.tgz", + "integrity": "sha512-jQL3lRnocaFtu3V00JToYz/4QkNWswxijDaCVNZRiRTO3HQDLsdu1ZtmIUvV4yPp+rvWm5j0y0TG/S61cuijTg==", + "dev": true, + "license": "MIT", + "dependencies": { + "tslib": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/use-composed-ref": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/use-composed-ref/-/use-composed-ref-1.4.0.tgz", + "integrity": "sha512-djviaxuOOh7wkj0paeO1Q/4wMZ8Zrnag5H6yBvzN7AKKe8beOaED9SF5/ByLqsku8NP4zQqsvM2u3ew/tJK8/w==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/use-isomorphic-layout-effect": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/use-isomorphic-layout-effect/-/use-isomorphic-layout-effect-1.2.1.tgz", + "integrity": "sha512-tpZZ+EX0gaghDAiFR37hj5MgY6ZN55kLiPkJsKxBMZ6GZdOSPJXiOzPM984oPYZ5AnehYx5WQp1+ME8I/P/pRA==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/use-latest": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/use-latest/-/use-latest-1.3.0.tgz", + "integrity": "sha512-mhg3xdm9NaM8q+gLT8KryJPnRFOz1/5XPBhmDEVZK1webPzDjrPk7f/mbpeLqTgB9msytYWANxgALOCJKnLvcQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "use-isomorphic-layout-effect": "^1.1.1" + }, + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/use-sidecar": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/use-sidecar/-/use-sidecar-1.1.3.tgz", + "integrity": "sha512-Fedw0aZvkhynoPYlA5WXrMCAMm+nSWdZt6lzJQ7Ok8S6Q+VsHmHpRWndVRJ8Be0ZbkfPc5LRYH+5XrzXcEeLRQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "detect-node-es": "^1.1.0", + "tslib": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/use-sync-external-store": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/use-sync-external-store/-/use-sync-external-store-1.7.0.tgz", + "integrity": "sha512-6L+EeigHMQhdaIPNIFUKwfWJSwWFQ8gJbJ2DLOs5sDIegTwR9fRxvnM3uciHKjIZhFz+KAv2emhWMRvDmMcY8A==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, + "node_modules/whatwg-mimetype": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-3.0.0.tgz", + "integrity": "sha512-nt+N2dzIutVRxARx1nghPKGv1xHikU7HKdfafKkLNLindmPU/ch3U31NOCGGA/dmPcmb1VlofO0vnKAcsm0o/Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/ws": { + "version": "8.21.3", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.3.tgz", + "integrity": "sha512-201TZ/kPWxoPr/OKWjquZR1SWKXcvxdH+e1xrx89b3YbmzLMFCLfnaG1HFIgWzJOEWZ7MvpK++odZufgYR50Rw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/zod": { + "version": "4.6.5", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.6.5.tgz", + "integrity": "sha512-v5l/aFXZQeai4awLbOpSoHecE9UiMrnfx75tEXLjNonXVARxQ5mOeipTjROUchszUNCqnE+hqAMujRsRHsut2Q==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/zustand": { + "version": "5.0.15", + "resolved": "https://registry.npmjs.org/zustand/-/zustand-5.0.15.tgz", + "integrity": "sha512-MpSEjRiBkA9crSYeOUH32rJC7SVqAbm0Fqcqge/bUi2PPoLcBWKOsG+C8mevmpr8TwXHBVkChbbJiyvkE+i/3A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.20.0" + }, + "peerDependencies": { + "@types/react": ">=18.0.0", + "immer": ">=9.0.6", + "react": ">=18.0.0", + "use-sync-external-store": ">=1.2.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "immer": { + "optional": true + }, + "react": { + "optional": true + }, + "use-sync-external-store": { + "optional": true + } + } + } + } +} diff --git a/chat-sdk/package.json b/chat-sdk/package.json new file mode 100644 index 0000000000..b13f75656a --- /dev/null +++ b/chat-sdk/package.json @@ -0,0 +1,108 @@ +{ + "name": "windmill-chat", + "description": "Build chat interfaces on Windmill flows deployed in chat mode, from any frontend or raw app", + "version": "1.810.0", + "author": "Ruben Fiszel", + "license": "Apache-2.0", + "homepage": "https://github.com/windmill-labs/windmill/tree/main/chat-sdk#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/windmill-labs/windmill.git", + "directory": "chat-sdk" + }, + "bugs": { + "url": "https://github.com/windmill-labs/windmill/issues" + }, + "keywords": [ + "windmill", + "chat", + "ai", + "agent", + "react" + ], + "sideEffects": false, + "type": "module", + "main": "dist/index.cjs", + "module": "dist/index.js", + "types": "dist/index.d.ts", + "exports": { + ".": { + "import": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + }, + "require": { + "types": "./dist/index.d.ts", + "default": "./dist/index.cjs" + } + }, + "./react": { + "import": { + "types": "./dist/react.d.ts", + "default": "./dist/react.js" + }, + "require": { + "types": "./dist/react.d.ts", + "default": "./dist/react.cjs" + } + }, + "./ai-sdk": { + "import": { + "types": "./dist/ai-sdk.d.ts", + "default": "./dist/ai-sdk.js" + }, + "require": { + "types": "./dist/ai-sdk.d.ts", + "default": "./dist/ai-sdk.cjs" + } + }, + "./assistant-ui": { + "import": { + "types": "./dist/assistant-ui.d.ts", + "default": "./dist/assistant-ui.js" + }, + "require": { + "types": "./dist/assistant-ui.d.ts", + "default": "./dist/assistant-ui.cjs" + } + } + }, + "files": [ + "dist", + "README.md" + ], + "scripts": { + "build": "tsdown && tsc -p tsconfig.build.json", + "check": "tsc --noEmit", + "test": "bun test" + }, + "peerDependencies": { + "@assistant-ui/react": ">=0.15", + "ai": ">=5", + "react": ">=18" + }, + "peerDependenciesMeta": { + "@assistant-ui/react": { + "optional": true + }, + "ai": { + "optional": true + }, + "react": { + "optional": true + } + }, + "devDependencies": { + "@ai-sdk/react": "^4.0.102", + "@assistant-ui/react": "^0.15.19", + "@happy-dom/global-registrator": "^20.14.5", + "@types/bun": "^1.3.0", + "@types/react": "^19.0.0", + "@types/react-dom": "^19.3.0", + "ai": "^7.0.99", + "react": "^19.0.0", + "react-dom": "^19.3.0", + "tsdown": "^0.12.9", + "typescript": "^5.4.5" + } +} diff --git a/chat-sdk/src/ai-sdk.ts b/chat-sdk/src/ai-sdk.ts new file mode 100644 index 0000000000..ba500446a3 --- /dev/null +++ b/chat-sdk/src/ai-sdk.ts @@ -0,0 +1,324 @@ +import type { ChatTransport, UIMessage, UIMessageChunk, UIMessagePart } from 'ai' +import { WindmillApiError, WindmillChatApi, type WindmillChatApiOptions } from './api' +import { followJob } from './follow' +import type { AgentStreamEvent } from './stream' +import type { ChatMessage, Conversation } from './types' +import { + conversationIdFor, + errorResultMessage, + extractChatAnswer, + isAbortError, + isErrorResult, + parseJsonOr, + randomId +} from './utils' + +export interface WindmillChatTransportOptions extends WindmillChatApiOptions { + /** Path of a deployed flow with chat mode enabled, e.g. `f/support/assistant`. */ + flowPath: string + /** Extra flow inputs sent with every message; `sendMessage(msg, { body })` adds per-message ones. */ + inputs?: Record +} + +export interface WindmillChatTransport + extends ChatTransport { + /** The Windmill conversation id behind an AI SDK chat id (a UUID chat id is used as is). */ + conversationId(chatId: string): string + /** Server history of a chat as `UIMessage`s, oldest first, for `useChat({ messages })`. Needs `flow_conversations:read`. */ + loadMessages(chatId: string, options?: { page?: number; perPage?: number }): Promise + /** The user's conversations for this flow, most recent first. */ + listConversations(options?: { page?: number; perPage?: number }): Promise + deleteConversation(chatId: string): Promise +} + +interface JobEntry { + jobId: string + offset?: number + done: boolean +} + +/** + * A Vercel AI SDK `ChatTransport` over a chat-mode flow: `useChat({ transport })` + * (and AI Elements, which builds on it) then work against Windmill unchanged. + * The chat id is the conversation, so a UUID id lines up with server history. + */ +export function createWindmillChatTransport( + options: WindmillChatTransportOptions +): WindmillChatTransport { + const api = new WindmillChatApi(options) + // One in-flight or finished job per chat, for `reconnectToStream`. + const jobs = new Map() + + return { + conversationId: conversationIdFor, + + async sendMessages({ chatId, messages, abortSignal, body }) { + const last = messages[messages.length - 1] + if (!last || last.role !== 'user') { + throw new Error('windmill-chat: the last message must be a user message') + } + if (last.parts.some((p) => p.type === 'file')) { + throw new Error( + 'windmill-chat: attachments are not supported; upload the file yourself and pass its reference through `body`' + ) + } + const text = last.parts + .filter((p): p is Extract, { type: 'text' }> => p.type === 'text') + .map((p) => p.text) + .join('\n') + const memoryId = conversationIdFor(chatId) + const jobId = await api.runFlow( + options.flowPath, + { ...options.inputs, ...(body as Record | undefined), user_message: text }, + { memoryId, signal: abortSignal } + ) + const entry: JobEntry = { jobId, done: false } + jobs.set(chatId, entry) + return chunkStream(api, entry, abortSignal) + }, + + async reconnectToStream({ chatId, abortSignal }) { + const entry = jobs.get(chatId) + if (!entry || entry.done) return null + return chunkStream(api, entry, abortSignal) + }, + + async loadMessages(chatId, pagination) { + // A chat that hasn't sent anything yet has no conversation on the server. + const rows = await api.listMessages(conversationIdFor(chatId), pagination).catch((e) => { + if (e instanceof WindmillApiError && e.status === 404) return [] + throw e + }) + return toUIMessages( + rows.map((row) => ({ + id: row.id, + serverId: row.id, + role: row.message_type, + content: row.content, + success: row.success ?? true, + createdAt: row.created_at, + jobId: row.job_id ?? undefined, + stepName: row.step_name ?? undefined, + pending: false, + seq: row.created_seq, + tool: toolFromRowContent(row.message_type, row.content, row.success ?? true) + })) + ) as UI_MESSAGE[] + }, + + async listConversations(pagination) { + const rows = await api.listConversations(options.flowPath, pagination) + return rows.map((row) => ({ + id: row.id, + title: row.title ?? undefined, + createdAt: row.created_at, + updatedAt: row.updated_at + })) + }, + + async deleteConversation(chatId) { + await api.deleteConversation(conversationIdFor(chatId)) + jobs.delete(chatId) + } + } +} + +function toolFromRowContent(role: string, content: string, success: boolean): ChatMessage['tool'] { + if (role !== 'tool') return undefined + const name = /^Used (.+) tool$/.exec(content)?.[1] ?? /^Error executing (.+)$/.exec(content)?.[1] + return name ? { name, status: success ? 'success' : 'error' } : undefined +} + +/** Streams a job's answer as AI SDK chunks; resumes from `entry.offset` when the job is already running. */ +function chunkStream( + api: WindmillChatApi, + entry: JobEntry, + signal: AbortSignal | undefined +): ReadableStream { + return new ReadableStream({ + async start(controller) { + const parts = new PartWriter((chunk) => controller.enqueue(chunk)) + parts.emit({ type: 'start' }) + try { + let failure: string | undefined + for await (const event of followJob(api, entry.jobId, { + signal, + streamOffset: entry.offset, + onOffset: (offset) => { + entry.offset = offset + } + })) { + if (event.type === 'stream') { + for (const e of event.events) parts.apply(e) + continue + } + parts.closeOpen() + failure = await failureText(api, entry.jobId, event.result, signal) + if (failure === undefined && !parts.streamedText) { + // No agent streamed: the flow's result is the answer. + const answer = extractChatAnswer(event.result) + if (answer !== undefined) { + parts.text(answer) + parts.closeOpen() + } + } + } + entry.done = true + parts.emit(failure === undefined ? { type: 'finish' } : { type: 'error', errorText: failure }) + } catch (e) { + if (!isAbortError(e)) { + parts.closeOpen() + parts.emit({ type: 'error', errorText: e instanceof Error ? e.message : String(e) }) + } + } finally { + controller.close() + } + } + }) +} + +/** A completed flow's error, when the job did fail (the envelope alone is a legitimate result). */ +async function failureText( + api: WindmillChatApi, + jobId: string, + result: unknown, + signal: AbortSignal | undefined +): Promise { + if (!isErrorResult(result)) return undefined + const failed = await api + .getCompletedResult(jobId, signal) + .then((r) => r.success === false) + .catch(() => true) + return failed ? errorResultMessage(result) : undefined +} + +/** + * Turns agent events into AI SDK chunks. Text and reasoning are open parts that a + * tool call closes (a new round starts new parts); tool calls are `dynamic-tool` + * parts, since the UI declares no tools of its own. + */ +class PartWriter { + streamedText = false + #textId: string | undefined + #reasoningId: string | undefined + #started = new Set() + #inputSent = new Set() + + constructor(readonly emit: (chunk: UIMessageChunk) => void) {} + + text(delta: string): void { + this.streamedText = true + if (!this.#textId) { + this.#textId = randomId() + this.emit({ type: 'text-start', id: this.#textId }) + } + this.emit({ type: 'text-delta', id: this.#textId, delta }) + } + + reasoning(delta: string): void { + if (!this.#reasoningId) { + this.#reasoningId = randomId() + this.emit({ type: 'reasoning-start', id: this.#reasoningId }) + } + this.emit({ type: 'reasoning-delta', id: this.#reasoningId, delta }) + } + + closeOpen(): void { + if (this.#reasoningId) { + this.emit({ type: 'reasoning-end', id: this.#reasoningId }) + this.#reasoningId = undefined + } + if (this.#textId) { + this.emit({ type: 'text-end', id: this.#textId }) + this.#textId = undefined + } + } + + apply(event: AgentStreamEvent): void { + switch (event.type) { + case 'token_delta': + this.text(event.content) + break + case 'reasoning_token_delta': + this.reasoning(event.content) + break + case 'tool_call': + this.closeOpen() + this.#toolStart(event.call_id, event.function_name) + break + case 'tool_call_arguments': + this.closeOpen() + this.#toolStart(event.call_id, event.function_name) + this.#inputSent.add(event.call_id) + this.emit({ + type: 'tool-input-available', + toolCallId: event.call_id, + toolName: event.function_name, + input: parseJsonOr(event.arguments), + dynamic: true + }) + break + case 'tool_execution': + this.closeOpen() + this.#toolStart(event.call_id, event.function_name) + break + case 'tool_result': + this.#toolStart(event.call_id, event.function_name) + if (!this.#inputSent.has(event.call_id)) { + this.#inputSent.add(event.call_id) + this.emit({ + type: 'tool-input-available', + toolCallId: event.call_id, + toolName: event.function_name, + input: undefined, + dynamic: true + }) + } + this.emit( + event.success + ? { type: 'tool-output-available', toolCallId: event.call_id, output: parseJsonOr(event.result), dynamic: true } + : { type: 'tool-output-error', toolCallId: event.call_id, errorText: event.result, dynamic: true } + ) + break + } + } + + #toolStart(callId: string, name: string): void { + if (this.#started.has(callId)) return + this.#started.add(callId) + this.emit({ type: 'tool-input-start', toolCallId: callId, toolName: name, dynamic: true }) + } +} + +/** + * `ChatMessage`s (Windmill's role-per-row model) as `UIMessage`s: an assistant + * turn becomes one message whose parts carry its text, reasoning and tool calls. + */ +export function toUIMessages(messages: ChatMessage[]): UIMessage[] { + const out: UIMessage[] = [] + for (const m of messages) { + if (m.role === 'user' || m.role === 'system') { + out.push({ id: m.id, role: m.role, parts: [{ type: 'text', text: m.content }] }) + continue + } + let target = out[out.length - 1] + if (!target || target.role !== 'assistant') { + target = { id: m.id, role: 'assistant', parts: [] } + out.push(target) + } + if (m.role === 'tool') { + const toolCallId = m.tool?.callId ?? m.id + const toolName = m.tool?.name ?? 'tool' + const input = parseJsonOr(m.tool?.arguments) + target.parts.push( + m.success + ? { type: 'dynamic-tool', toolName, toolCallId, state: 'output-available', input, output: parseJsonOr(m.tool?.result) ?? m.content } + : { type: 'dynamic-tool', toolName, toolCallId, state: 'output-error', input, errorText: m.tool?.result ?? m.content } + ) + continue + } + if (m.reasoning) target.parts.push({ type: 'reasoning', text: m.reasoning, state: 'done' }) + if (m.content) target.parts.push({ type: 'text', text: m.content, state: 'done' }) + } + return out +} diff --git a/chat-sdk/src/api.ts b/chat-sdk/src/api.ts new file mode 100644 index 0000000000..0570154ec2 --- /dev/null +++ b/chat-sdk/src/api.ts @@ -0,0 +1,294 @@ +import type { FetchLike, TokenSource } from './types' + +export interface WindmillChatApiOptions { + baseUrl: string + workspace: string + /** Omit to rely on the session cookie of the Windmill origin. */ + token?: TokenSource + fetch?: FetchLike +} + +export class WindmillApiError extends Error { + constructor( + message: string, + readonly status: number + ) { + super(message) + this.name = 'WindmillApiError' + } +} + +export interface FlowConversation { + id: string + workspace_id: string + flow_path: string + title?: string | null + created_at: string + updated_at: string + created_by: string +} + +export interface FlowConversationMessage { + id: string + conversation_id: string + message_type: 'user' | 'assistant' | 'system' | 'tool' + content: string + job_id?: string | null + created_at: string + created_seq: number + step_name?: string | null + success?: boolean +} + +export type JobUpdateEvent = + | { + type: 'update' + running?: boolean + completed?: boolean + new_result_stream?: string + stream_offset?: number + only_result?: unknown + flow_stream_job_id?: string + } + | { type: 'error'; error: string } + | { type: 'notfound' } + | { type: 'timeout' } + | { type: 'ping' } + +export interface CompletedJobResult { + completed: boolean + success?: boolean + result?: unknown +} + +/** The part of a flow job's status that names the jobs its steps ran as. */ +export interface FlowJobStatus { + flow_status?: { + modules?: FlowStepStatus[] | null + failure_module?: FlowStepStatus | null + preprocessor_module?: FlowStepStatus | null + } | null +} + +export interface FlowStepStatus { + job?: string | null + flow_jobs?: string[] | null +} + +/** Thin client over the Windmill endpoints a chat-mode flow uses. */ +export class WindmillChatApi { + readonly #baseUrl: string + readonly #workspace: string + readonly #token: TokenSource | undefined + readonly #fetch: FetchLike + + constructor(options: WindmillChatApiOptions) { + this.#baseUrl = normalizeBaseUrl(options.baseUrl) + this.#workspace = options.workspace + this.#token = options.token + this.#fetch = options.fetch ?? ((input, init) => globalThis.fetch(input, init)) + } + + /** Starts a turn: runs the flow with `memory_id` set to the conversation id. Returns the job id. */ + async runFlow( + flowPath: string, + args: Record, + options: { memoryId: string; signal?: AbortSignal } + ): Promise { + const res = await this.#request(`jobs/run/f/${encodePath(flowPath)}`, { + method: 'POST', + query: { memory_id: options.memoryId, skip_preprocessor: 'true' }, + body: args, + signal: options.signal + }) + return (await res.text()).trim() + } + + /** + * One server-sent-events connection to a job's updates. The server closes it after + * `TIMEOUT_SSE_STREAM` (a `timeout` event); resume by calling again with the last + * `stream_offset`, never by re-running the flow. + */ + async *streamJob( + jobId: string, + options: { streamOffset?: number; signal?: AbortSignal } = {} + ): AsyncGenerator { + const query: Record = { fast: 'true', only_result: 'true' } + if (options.streamOffset !== undefined) { + query.stream_offset = String(options.streamOffset) + } + const res = await this.#request(`jobs_u/getupdate_sse/${encodeURIComponent(jobId)}`, { + query, + accept: 'text/event-stream', + signal: options.signal + }) + if (!res.body) { + throw new WindmillApiError('The job update stream has no body', res.status) + } + for await (const data of readServerSentEvents(res.body)) { + try { + yield JSON.parse(data) as JobUpdateEvent + } catch { + // A frame that isn't JSON carries nothing the chat can use. + } + } + } + + async getCompletedResult(jobId: string, signal?: AbortSignal): Promise { + const res = await this.#request( + `jobs_u/completed/get_result_maybe/${encodeURIComponent(jobId)}`, + { signal } + ) + return (await res.json()) as CompletedJobResult + } + + /** A flow job with its status: the step job ids are what persisted messages carry as `job_id`. */ + async getFlowJob(jobId: string, signal?: AbortSignal): Promise { + const res = await this.#request(`jobs_u/get/${encodeURIComponent(jobId)}`, { + query: { no_logs: 'true' }, + signal + }) + return (await res.json()) as FlowJobStatus + } + + async cancelJob(jobId: string, reason = 'Stopped from the chat'): Promise { + await this.#request(`jobs_u/queue/cancel/${encodeURIComponent(jobId)}`, { + method: 'POST', + body: { reason } + }) + } + + async listConversations( + flowPath: string, + options: { page?: number; perPage?: number; signal?: AbortSignal } = {} + ): Promise { + const res = await this.#request('flow_conversations/list', { + query: pagination(options, { flow_path: flowPath }), + signal: options.signal + }) + return (await res.json()) as FlowConversation[] + } + + /** + * Without `afterSeq`: one page counted from the newest message, returned oldest first. + * With `afterSeq`: the messages created after that cursor, oldest first. + */ + async listMessages( + conversationId: string, + options: { page?: number; perPage?: number; afterSeq?: number; signal?: AbortSignal } = {} + ): Promise { + const extra: Record = {} + if (options.afterSeq !== undefined) extra.after_seq = String(options.afterSeq) + const res = await this.#request( + `flow_conversations/${encodeURIComponent(conversationId)}/messages`, + { query: pagination(options, extra), signal: options.signal } + ) + return (await res.json()) as FlowConversationMessage[] + } + + async deleteConversation(conversationId: string): Promise { + await this.#request(`flow_conversations/delete/${encodeURIComponent(conversationId)}`, { + method: 'DELETE' + }) + } + + async #request( + path: string, + init: { + method?: string + query?: Record + body?: unknown + accept?: string + signal?: AbortSignal + } = {} + ): Promise { + const url = new URL(`${this.#baseUrl}/api/w/${encodeURIComponent(this.#workspace)}/${path}`) + for (const [k, v] of Object.entries(init.query ?? {})) url.searchParams.set(k, v) + + const headers: Record = {} + if (init.accept) headers['Accept'] = init.accept + if (init.body !== undefined) headers['Content-Type'] = 'application/json' + const token = typeof this.#token === 'function' ? await this.#token() : this.#token + if (token) headers['Authorization'] = `Bearer ${token}` + + const res = await this.#fetch(url.toString(), { + method: init.method ?? 'GET', + headers, + body: init.body === undefined ? undefined : JSON.stringify(init.body), + // A token must not be paired with ambient cookies; without one, the cookie is + // the credential and only rides same-origin requests. + credentials: token ? 'omit' : 'same-origin', + signal: init.signal + }) + if (!res.ok) { + const text = await res.text().catch(() => '') + throw new WindmillApiError( + `${init.method ?? 'GET'} ${path} failed (${res.status})${text ? `: ${text}` : ''}`, + res.status + ) + } + return res + } +} + +export function normalizeBaseUrl(baseUrl: string): string { + return baseUrl.replace(/\/+$/, '').replace(/\/api$/, '') +} + +function encodePath(path: string): string { + return path.split('/').map(encodeURIComponent).join('/') +} + +function pagination( + options: { page?: number; perPage?: number }, + extra: Record +): Record { + const query = { ...extra } + if (options.page !== undefined) query.page = String(options.page) + if (options.perPage !== undefined) query.per_page = String(options.perPage) + return query +} + +/** Yields the `data` payload of each event in a `text/event-stream` body. */ +export async function* readServerSentEvents( + body: ReadableStream +): AsyncGenerator { + const reader = body.getReader() + const decoder = new TextDecoder() + let buffer = '' + // A CR ending a chunk may be half of a CRLF; it waits for the next chunk. + let carry = '' + try { + while (true) { + const { value, done } = await reader.read() + if (done) break + let text = carry + decoder.decode(value, { stream: true }) + carry = '' + if (text.endsWith('\r')) { + carry = '\r' + text = text.slice(0, -1) + } + buffer += text.replace(/\r\n?/g, '\n') + let end: number + while ((end = buffer.indexOf('\n\n')) !== -1) { + const data = eventData(buffer.slice(0, end)) + buffer = buffer.slice(end + 2) + if (data !== undefined) yield data + } + } + if (carry) buffer += '\n' + const data = eventData(buffer) + if (data !== undefined) yield data + } finally { + // Closes the connection when the consumer stops early. + reader.cancel().catch(() => {}) + } +} + +function eventData(block: string): string | undefined { + const lines = block + .split('\n') + .filter((line) => line.startsWith('data:')) + .map((line) => line.slice(line.startsWith('data: ') ? 6 : 5)) + return lines.length > 0 ? lines.join('\n') : undefined +} diff --git a/chat-sdk/src/assistant-ui.ts b/chat-sdk/src/assistant-ui.ts new file mode 100644 index 0000000000..ef22f28531 --- /dev/null +++ b/chat-sdk/src/assistant-ui.ts @@ -0,0 +1,124 @@ +import { + useExternalStoreRuntime, + type AppendMessage, + type AssistantRuntime, + type ExternalStoreAdapter, + type ThreadMessageLike +} from '@assistant-ui/react' +import { useEffect, useMemo } from 'react' +import { useWindmillChat } from './react' +import type { ChatMessage, ChatOptions } from './types' +import { parseJsonOr } from './utils' + +/** One assistant turn: the assistant and tool rows between two user messages. */ +export interface WindmillTurn { + id: string + role: 'user' | 'assistant' | 'system' + messages: ChatMessage[] +} + +export interface WindmillRuntimeOptions extends ChatOptions { + /** Load the conversation list on mount so `ThreadListPrimitive` has something to show. Default true. */ + threadList?: boolean +} + +/** + * An assistant-ui runtime over a chat-mode flow, for `AssistantRuntimeProvider`. + * Conversations become threads, so the thread list primitives switch, create and + * delete Windmill conversations. + */ +export function useWindmillRuntime(options: WindmillRuntimeOptions): AssistantRuntime { + const chat = useWindmillChat(options) + const turns = useMemo(() => groupTurns(chat.messages), [chat.messages]) + const withThreadList = options.threadList !== false + useEffect(() => { + if (withThreadList) chat.loadConversations().catch(() => {}) + }, [chat.chat, withThreadList]) + + const adapter: ExternalStoreAdapter = { + messages: turns, + isRunning: chat.status === 'submitted' || chat.status === 'streaming', + convertMessage: toThreadMessage, + onNew: async (message: AppendMessage) => { + if (message.role !== 'user') return + await chat.sendMessage(appendedText(message)) + }, + onCancel: async () => { + await chat.stop() + }, + adapters: withThreadList + ? { + threadList: { + threadId: chat.conversationId, + threads: chat.conversations.map((c) => ({ status: 'regular' as const, id: c.id, title: c.title })), + onSwitchToNewThread: () => chat.newConversation(), + onSwitchToThread: (id) => chat.selectConversation(id), + onDelete: (id) => chat.deleteConversation(id) + } + } + : undefined + } + return useExternalStoreRuntime(adapter) +} + +function appendedText(message: AppendMessage): string { + return message.content + .filter((p): p is { type: 'text'; text: string } => p.type === 'text') + .map((p) => p.text) + .join('\n') +} + +/** Folds the role-per-row message list into turns: one entry per user message, one per answer. */ +export function groupTurns(messages: ChatMessage[]): WindmillTurn[] { + const turns: WindmillTurn[] = [] + for (const m of messages) { + const last = turns[turns.length - 1] + if (m.role === 'user' || m.role === 'system' || !last || last.role !== 'assistant') { + turns.push({ id: m.id, role: m.role === 'tool' ? 'assistant' : m.role, messages: [m] }) + } else { + last.messages.push(m) + } + } + return turns +} + +/** A turn as assistant-ui content parts; the status reflects streaming and a failed flow. */ +export function toThreadMessage(turn: WindmillTurn): ThreadMessageLike { + const first = turn.messages[0] + const createdAt = new Date(first.createdAt) + if (turn.role !== 'assistant') { + return { id: turn.id, role: turn.role, createdAt, content: [{ type: 'text', text: first.content }] } + } + const content: ThreadContentPart[] = [] + for (const m of turn.messages) { + if (m.role === 'tool') { + const args = parseJsonOr(m.tool?.arguments) + content.push({ + type: 'tool-call', + toolCallId: m.tool?.callId ?? m.id, + toolName: m.tool?.name ?? 'tool', + args: (isJsonObject(args) ? args : args === undefined ? {} : { input: args }) as ToolCallArgs, + argsText: m.tool?.arguments ?? '', + result: m.tool?.status === 'running' ? undefined : (parseJsonOr(m.tool?.result) ?? m.content), + isError: m.tool?.status === 'error' + }) + continue + } + if (m.reasoning) content.push({ type: 'reasoning', text: m.reasoning }) + if (m.content) content.push({ type: 'text', text: m.content }) + } + const last = turn.messages[turn.messages.length - 1] + const status: ThreadMessageLike['status'] = turn.messages.some((m) => m.pending) + ? { type: 'running' } + : last.role === 'assistant' && !last.success + ? { type: 'incomplete', reason: 'error', error: last.content } + : { type: 'complete', reason: 'stop' } + return { id: turn.id, role: 'assistant', createdAt, content, status } +} + +type ThreadContentPart = Exclude[number] +type ToolCallArgs = Extract['args'] + +function isJsonObject(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} diff --git a/chat-sdk/src/chat.ts b/chat-sdk/src/chat.ts new file mode 100644 index 0000000000..866d185d84 --- /dev/null +++ b/chat-sdk/src/chat.ts @@ -0,0 +1,744 @@ +import { + WindmillApiError, + WindmillChatApi, + type FlowConversation, + type FlowConversationMessage +} from './api' +import { resolveConfig, type ResolvedConfig } from './config' +import { followJob } from './follow' +import { createLocalHistory, type LocalHistory } from './history' +import type { AgentStreamEvent } from './stream' +import type { + Chat, + ChatMessage, + ChatOptions, + ChatState, + Conversation, + ToolInvocation +} from './types' +import { + conversationTitle, + errorResultMessage, + extractChatAnswer, + isAbortError, + isErrorResult, + now, + randomId, + sleep +} from './utils' + +const POLL_INTERVAL_MS = 1000 +/** Local history mirrors the state; a stream of deltas is coalesced into one write. */ +const PERSIST_DEBOUNCE_MS = 250 +/** Messages persist from spawned tasks that can land just after the flow completes. */ +const RECONCILE_ATTEMPTS = 3 +const RECONCILE_DELAY_MS = 400 + +interface Turn { + controller: AbortController + conversationId: string + /** Id of the turn's user message; the answer is whatever follows it. */ + userMessageId: string + jobId?: string + /** The flow job and its step jobs; a persisted answer carries one of them as `job_id`. */ + jobIds?: Set + /** Id of the streaming assistant message; cleared when a tool call ends the round. */ + assistantId?: string + streamedText: boolean +} + +export function createChat(options: ChatOptions): Chat { + return new ChatImpl(options) +} + +class ChatImpl implements Chat { + readonly #config: ResolvedConfig + readonly #api: WindmillChatApi + readonly #local: LocalHistory + readonly #listeners = new Set<(state: ChatState) => void>() + #state: ChatState + #turn: Turn | undefined + #page = 1 + #persistTimer: ReturnType | undefined + + constructor(options: ChatOptions) { + this.#config = resolveConfig(options) + this.#api = new WindmillChatApi({ + baseUrl: this.#config.baseUrl, + workspace: this.#config.workspace, + token: this.#config.token, + fetch: this.#config.fetch + }) + this.#local = createLocalHistory( + this.#config.storage, + `windmill-chat:${this.#config.baseUrl}:${this.#config.workspace}:${this.#config.flowPath}` + + (this.#config.storageKey ? `:${this.#config.storageKey}` : '') + ) + this.#state = { + conversationId: undefined, + messages: [], + status: 'idle', + error: undefined, + conversations: [], + history: this.#config.history, + loadingMessages: false, + hasMoreMessages: false + } + } + + getState = (): ChatState => this.#state + + subscribe = (listener: (state: ChatState) => void): (() => void) => { + this.#listeners.add(listener) + listener(this.#state) + return () => { + this.#listeners.delete(listener) + } + } + + sendMessage = async ( + text: string, + options: { inputs?: Record } = {} + ): Promise => { + const content = text.trim() + if (!content) return + if (this.#turn) { + throw new Error('windmill-chat: a message is already being answered; call stop() first') + } + const isNew = this.#state.conversationId === undefined + const conversationId = this.#state.conversationId ?? randomId() + const turn: Turn = { + controller: new AbortController(), + conversationId, + userMessageId: `pending-${randomId()}`, + streamedText: false + } + this.#turn = turn + + const timestamp = now() + const conversation: Conversation = this.#state.conversations.find( + (c) => c.id === conversationId + ) ?? { id: conversationId, title: conversationTitle(content), createdAt: timestamp, updatedAt: timestamp } + const touched = { ...conversation, updatedAt: timestamp } + this.#set({ + conversationId, + conversations: [touched, ...this.#state.conversations.filter((c) => c.id !== conversationId)], + messages: [ + ...this.#state.messages, + { id: turn.userMessageId, role: 'user', content, success: true, createdAt: timestamp, pending: true } + ], + status: 'submitted', + error: undefined + }) + this.#rememberConversation() + + try { + turn.jobId = await this.#api.runFlow( + this.#config.flowPath, + { ...this.#config.inputs, ...options.inputs, user_message: content }, + { memoryId: conversationId, signal: turn.controller.signal } + ) + const stopPolling = this.#state.history === 'server' ? this.#startPolling(turn) : () => {} + let result: unknown + try { + result = await this.#follow(turn, stopPolling) + } finally { + stopPolling() + } + await this.#finishTurn(turn, result, isNew) + } catch (e) { + // stop() and a conversation switch abort the turn and settle the state themselves. + if (turn.controller.signal.aborted || isAbortError(e)) return + this.#failTurn(turn, e) + } finally { + if (this.#turn === turn) this.#turn = undefined + } + } + + stop = async (): Promise => { + const turn = this.#turn + if (!turn) return + this.#detachTurn() + if (this.#state.conversationId === turn.conversationId) { + this.#set({ messages: finalized(this.#state.messages), status: 'idle' }) + this.#persistLocal() + } + if (turn.jobId) { + // Needs `jobs:write` on a token; the stream is closed either way. + await this.#api.cancelJob(turn.jobId).catch(() => {}) + } + if (this.#state.history !== 'server') return + // A cancelled flow persists its failure as the assistant's answer. Picked up only + // while the conversation is still idle: a turn started meanwhile owns the state. + await sleep(RECONCILE_DELAY_MS).catch(() => {}) + if (this.#turn || this.#state.conversationId !== turn.conversationId) return + await this.#syncFromServer(turn.conversationId).catch(() => {}) + } + + newConversation = (): void => { + this.#leaveConversation() + this.#page = 1 + this.#set({ + conversationId: undefined, + messages: [], + status: 'idle', + error: undefined, + loadingMessages: false, + hasMoreMessages: false + }) + } + + selectConversation = async (conversationId: string): Promise => { + if (conversationId === this.#state.conversationId) return + this.#leaveConversation() + this.#page = 1 + this.#set({ + conversationId, + messages: [], + status: 'idle', + error: undefined, + loadingMessages: true, + hasMoreMessages: false + }) + if (this.#state.history !== 'server') { + this.#set({ + messages: this.#state.history === 'local' ? this.#local.getMessages(conversationId) : [], + loadingMessages: false + }) + return + } + try { + const rows = await this.#api.listMessages(conversationId, { + perPage: this.#config.pageSize + }) + if (this.#state.conversationId !== conversationId) return + this.#set({ + messages: rows.map(fromRow), + loadingMessages: false, + hasMoreMessages: rows.length === this.#config.pageSize + }) + } catch (e) { + if (this.#state.conversationId !== conversationId) return + if (this.#fallBackToLocal(e)) { + this.#set({ messages: this.#local.getMessages(conversationId), loadingMessages: false }) + return + } + this.#set({ loadingMessages: false, status: 'error', error: toError(e) }) + } + } + + loadConversations = async ( + options: { page?: number; perPage?: number } = {} + ): Promise => { + const page = options.page ?? 1 + let conversations: Conversation[] + if (this.#state.history === 'server') { + try { + const rows = await this.#api.listConversations(this.#config.flowPath, { + page, + perPage: options.perPage ?? this.#config.pageSize + }) + conversations = rows.map(fromConversation) + } catch (e) { + if (!this.#fallBackToLocal(e)) throw e + conversations = this.#local.listConversations() + } + } else { + conversations = this.#state.history === 'local' ? this.#local.listConversations() : [] + } + const known = new Set(this.#state.conversations.map((c) => c.id)) + this.#set({ + conversations: + page === 1 + ? conversations + : [...this.#state.conversations, ...conversations.filter((c) => !known.has(c.id))] + }) + return conversations + } + + deleteConversation = async (conversationId: string): Promise => { + if (this.#state.conversationId === conversationId) { + // Nothing of the current turn may be written back under the deleted id. + this.#detachTurn() + clearTimeout(this.#persistTimer) + this.#persistTimer = undefined + this.newConversation() + } + if (this.#state.history === 'server') { + await this.#api.deleteConversation(conversationId) + } else if (this.#state.history === 'local') { + this.#local.deleteConversation(conversationId) + } + this.#set({ conversations: this.#state.conversations.filter((c) => c.id !== conversationId) }) + } + + loadOlderMessages = async (): Promise => { + const conversationId = this.#state.conversationId + if ( + !conversationId || + this.#state.history !== 'server' || + !this.#state.hasMoreMessages || + this.#state.loadingMessages + ) { + return + } + const page = this.#page + 1 + this.#set({ loadingMessages: true }) + try { + const rows = await this.#api.listMessages(conversationId, { + page, + perPage: this.#config.pageSize + }) + if (this.#state.conversationId !== conversationId) return + const known = new Set(this.#state.messages.map((m) => m.serverId ?? m.id)) + this.#page = page + this.#set({ + messages: [...rows.map(fromRow).filter((m) => !known.has(m.id)), ...this.#state.messages], + hasMoreMessages: rows.length === this.#config.pageSize + }) + } finally { + if (this.#state.conversationId === conversationId) this.#set({ loadingMessages: false }) + } + } + + destroy = (): void => { + this.#leaveConversation() + } + + // ---- turn internals ---- + + async #follow(turn: Turn, onStreamStart: () => void): Promise { + let started = false + for await (const event of followJob(this.#api, turn.jobId!, { signal: turn.controller.signal })) { + if (event.type === 'completed') return event.result + if (!started) { + started = true + // Persisted rows for the streaming step would duplicate what is streaming. + onStreamStart() + } + this.#applyEvents(turn, event.events) + } + throw new Error('windmill-chat: the job stream ended before the flow completed') + } + + #applyEvents(turn: Turn, events: AgentStreamEvent[]): void { + if (events.length === 0 || !this.#turnActive(turn)) return + let messages = [...this.#state.messages] + const upsertTool = ( + callId: string, + name: string, + patch: Partial & { content?: string; success?: boolean } + ) => { + const { content, success, ...toolPatch } = patch + // Only this turn's tool messages are pending; a provider may reuse call ids across turns. + const i = messages.findIndex( + (m) => m.pending && m.role === 'tool' && m.tool?.callId === callId + ) + if (i >= 0) { + const existing = messages[i] + messages[i] = { + ...existing, + content: content ?? existing.content, + success: success ?? existing.success, + tool: { ...existing.tool!, ...toolPatch } + } + } else { + messages.push({ + id: `pending-${randomId()}`, + role: 'tool', + content: content ?? '', + success: success ?? true, + createdAt: now(), + pending: true, + tool: { callId, name, status: 'running', ...toolPatch } + }) + } + } + const appendAssistant = (text: string, reasoning: string) => { + const i = turn.assistantId + ? messages.findIndex((m) => m.id === turn.assistantId) + : -1 + if (i >= 0) { + const m = messages[i] + messages[i] = { + ...m, + content: m.content + text, + reasoning: reasoning ? (m.reasoning ?? '') + reasoning : m.reasoning + } + } else { + turn.assistantId = `pending-${randomId()}` + messages.push({ + id: turn.assistantId, + role: 'assistant', + content: text, + reasoning: reasoning || undefined, + success: true, + createdAt: now(), + pending: true + }) + } + } + for (const event of events) { + switch (event.type) { + case 'token_delta': + turn.streamedText = true + appendAssistant(event.content, '') + break + case 'reasoning_token_delta': + appendAssistant('', event.content) + break + case 'tool_call': + // The round's text is complete; text after the tool result is a new message. + turn.assistantId = undefined + upsertTool(event.call_id, event.function_name, { status: 'running' }) + break + case 'tool_call_arguments': + turn.assistantId = undefined + upsertTool(event.call_id, event.function_name, { arguments: event.arguments }) + break + case 'tool_execution': + turn.assistantId = undefined + upsertTool(event.call_id, event.function_name, { status: 'running' }) + break + case 'tool_result': + upsertTool(event.call_id, event.function_name, { + status: event.success ? 'success' : 'error', + result: event.result, + success: event.success, + // The same text Windmill persists for the tool message. + content: event.success + ? `Used ${event.function_name} tool` + : `Error executing ${event.function_name}` + }) + break + } + } + this.#set({ messages, status: 'streaming' }) + } + + async #finishTurn(turn: Turn, result: unknown, isNew: boolean): Promise { + if (!this.#turnActive(turn)) return + if (this.#state.history === 'server') { + turn.jobIds = await this.#turnJobIds(turn) + if (!this.#turnActive(turn)) return + const reconciled = await this.#reconcileTurn(turn) + if (!this.#turnActive(turn)) return + if (reconciled) { + this.#set({ status: 'idle' }) + this.#config.onFinish?.({ conversationId: turn.conversationId, jobId: turn.jobId, messages: this.#state.messages }) + if (isNew) await this.loadConversations().catch(() => {}) + return + } + // Server history just proved unreadable: the turn completes as local history. + } + let messages = this.#state.messages + let failed = false + if (isErrorResult(result)) { + // The envelope is also a legitimate result shape; the job's own status decides. + failed = await this.#api + .getCompletedResult(turn.jobId!, turn.controller.signal) + .then((r) => r.success === false) + .catch(() => true) + if (!this.#turnActive(turn)) return + if (failed) { + messages = [...messages, assistantMessage(errorResultMessage(result), false, turn.jobId)] + } + } + if (!failed && !turn.streamedText) { + const answer = extractChatAnswer(result) + if (answer !== undefined) { + messages = [...messages, assistantMessage(answer, true, turn.jobId)] + } + } + this.#set({ messages: finalized(messages), status: 'idle' }) + this.#persistLocal() + this.#config.onFinish?.({ conversationId: turn.conversationId, jobId: turn.jobId, messages: this.#state.messages }) + } + + /** + * Folds what the server persisted for the turn into the message list. The rows + * are written by the worker in their own transactions, each of which can trail + * the flow's completion, so a streamed message whose row hasn't landed stays and + * the list is re-read a few times before the rest is kept as streamed. + * Returns false when the server holds no answer for the turn: history fell back to + * local, the read was refused or kept failing, or no assistant row has landed. The + * caller then finishes the turn from the flow result, so an answer is never lost + * to history. Whether a row counts is read from the message list, not from what + * this read returned: the turn's polling may have merged the answer already. + */ + async #reconcileTurn(turn: Turn): Promise { + for (let attempt = 1; attempt <= RECONCILE_ATTEMPTS; attempt++) { + let rows: FlowConversationMessage[] + try { + rows = await this.#api.listMessages(turn.conversationId, { + afterSeq: this.#lastSeq(), + perPage: 100, + signal: turn.controller.signal + }) + } catch (e) { + if (isAbortError(e)) throw e + if (this.#fallBackToLocal(e)) return false + const refused = e instanceof WindmillApiError && (e.status === 401 || e.status === 403) + if (refused || attempt === RECONCILE_ATTEMPTS) return this.#answered(turn) + await sleep(RECONCILE_DELAY_MS, turn.controller.signal) + continue + } + if (!this.#turnActive(turn)) return true + this.#mergeRows(rows) + if (this.#answered(turn) && !this.#state.messages.some((m) => m.pending && m.content)) break + if (attempt < RECONCILE_ATTEMPTS) await sleep(RECONCILE_DELAY_MS, turn.controller.signal) + } + if (!this.#turnActive(turn)) return true + this.#set({ messages: finalized(this.#state.messages) }) + return this.#answered(turn) + } + + /** + * A persisted assistant message written by one of the turn's jobs follows the + * turn's user message. Tool rows alone are not an answer, and neither is a row + * from an earlier turn whose job outlived `stop()` (a token without `jobs:write` + * cannot cancel it), which can land after this turn's user row. + */ + #answered(turn: Turn): boolean { + const messages = this.#state.messages + const from = messages.findIndex((m) => m.id === turn.userMessageId) + const ownJob = (m: ChatMessage) => + turn.jobIds === undefined || (m.jobId !== undefined && turn.jobIds.has(m.jobId)) + return messages.some((m, i) => i > from && m.role === 'assistant' && m.seq !== undefined && ownJob(m)) + } + + /** + * The flow job plus every step job it ran, the failure and preprocessor steps + * included (a failure handler's answer is persisted under its own job). Unknown + * when the read fails. + */ + async #turnJobIds(turn: Turn): Promise | undefined> { + try { + const job = await this.#api.getFlowJob(turn.jobId!, turn.controller.signal) + const ids = new Set([turn.jobId!]) + const status = job.flow_status + for (const m of [...(status?.modules ?? []), status?.failure_module, status?.preprocessor_module]) { + if (m?.job) ids.add(m.job) + for (const j of m?.flow_jobs ?? []) ids.add(j) + } + return ids + } catch (e) { + if (isAbortError(e)) throw e + return undefined + } + } + + #failTurn(turn: Turn, e: unknown): void { + if (!this.#turnActive(turn)) return + const error = toError(e) + this.#set({ + messages: [...finalized(this.#state.messages), assistantMessage(error.message, false, turn.jobId)], + status: 'error', + error + }) + this.#persistLocal() + this.#config.onError?.(error, { conversationId: turn.conversationId, jobId: turn.jobId }) + } + + /** Before the answer streams, earlier steps may already have persisted messages. */ + #startPolling(turn: Turn): () => void { + let stopped = false + const { signal } = turn.controller + const loop = async () => { + while (!stopped) { + try { + await sleep(POLL_INTERVAL_MS, signal) + } catch { + return + } + if (stopped) return + try { + const rows = await this.#api.listMessages(turn.conversationId, { + afterSeq: this.#lastSeq(), + perPage: 100, + signal + }) + if (!stopped && this.#turnActive(turn)) this.#mergeRows(rows) + } catch { + // transient; the completion reconciliation catches up + } + } + } + void loop() + return () => { + stopped = true + } + } + + async #syncFromServer(conversationId: string): Promise { + const rows = await this.#api.listMessages(conversationId, { + afterSeq: this.#lastSeq(), + perPage: 100 + }) + if (this.#turn || this.#state.conversationId !== conversationId) return + this.#mergeRows(rows) + this.#set({ messages: finalized(this.#state.messages) }) + } + + /** + * Folds persisted rows into the message list. A row standing for a message the + * client already shows (same role and text; for a tool, the same tool name, since + * the server words a failure differently) takes its place under the client's id + * and keeps what only the stream knew: reasoning, call id, arguments, result. + * Other rows append in server order. Nothing is dropped: a streamed message + * outlives a row that never lands. + */ + #mergeRows(rows: FlowConversationMessage[]): void { + if (rows.length === 0) return + const messages = [...this.#state.messages] + const known = new Set(messages.map((m) => m.serverId ?? m.id)) + for (const row of rows.map(fromRow)) { + if (known.has(row.id)) continue + known.add(row.id) + const i = messages.findIndex( + (m) => + m.seq === undefined && + m.role === row.role && + (m.content === row.content || (row.tool !== undefined && m.tool?.name === row.tool.name)) + ) + if (i >= 0) { + const m = messages[i] + messages[i] = { + ...row, + id: m.id, + reasoning: m.reasoning ?? row.reasoning, + tool: m.tool ? { ...m.tool, status: row.tool?.status ?? m.tool.status } : row.tool + } + } else { + messages.push(row) + } + } + this.#set({ messages }) + } + + #lastSeq(): number | undefined { + let last: number | undefined + for (const m of this.#state.messages) { + if (m.seq !== undefined && (last === undefined || m.seq > last)) last = m.seq + } + return last + } + + /** Writes the current messages to local history; the conversation entry itself is `#rememberConversation`'s. */ + #persistLocal(): void { + clearTimeout(this.#persistTimer) + this.#persistTimer = undefined + const id = this.#state.conversationId + if (this.#state.history !== 'local' || !id) return + this.#local.saveMessages(id, this.#state.messages) + } + + /** + * Puts the current conversation at the head of local history. Only a turn moves + * a conversation there: merely viewing one must not reorder the list. + */ + #rememberConversation(): void { + const id = this.#state.conversationId + if (this.#state.history !== 'local' || !id) return + const conversation = this.#state.conversations.find((c) => c.id === id) + if (conversation) this.#local.upsertConversation(conversation) + } + + /** Whether an unreadable server history should silently become local history. */ + #fallBackToLocal(e: unknown): boolean { + if (this.#state.history !== 'server' || this.#config.historyExplicit) return false + if (e instanceof WindmillApiError && (e.status === 401 || e.status === 403)) { + this.#set({ history: 'local' }) + // The conversation now lives in the browser; list it there like one started local. + this.#rememberConversation() + return true + } + return false + } + + #turnActive(turn: Turn): boolean { + return this.#turn === turn && this.#state.conversationId === turn.conversationId + } + + /** Stops following the current answer; the flow itself keeps running. */ + #detachTurn(): void { + const turn = this.#turn + if (!turn) return + this.#turn = undefined + turn.controller.abort() + } + + /** + * Leaves the current conversation (for another one, or because the page goes + * away). A turn still in flight is detached and what it showed so far is kept, + * written out now rather than on the debounce that may never fire. + */ + #leaveConversation(): void { + if (this.#turn) { + this.#detachTurn() + this.#set({ messages: finalized(this.#state.messages), status: 'idle' }) + } + if (this.#persistTimer) this.#persistLocal() + } + + #set(patch: Partial): void { + this.#state = { ...this.#state, ...patch } + for (const listener of this.#listeners) listener(this.#state) + if (this.#state.history === 'local' && this.#state.conversationId) { + clearTimeout(this.#persistTimer) + this.#persistTimer = setTimeout(() => this.#persistLocal(), PERSIST_DEBOUNCE_MS) + } + } +} + +function fromRow(row: FlowConversationMessage): ChatMessage { + const toolName = + row.message_type === 'tool' + ? /^Used (.+) tool$/.exec(row.content)?.[1] ?? /^Error executing (.+)$/.exec(row.content)?.[1] + : undefined + const success = row.success ?? true + return { + id: row.id, + serverId: row.id, + role: row.message_type, + content: row.content, + success, + createdAt: row.created_at, + jobId: row.job_id ?? undefined, + stepName: row.step_name ?? undefined, + pending: false, + seq: row.created_seq, + tool: toolName ? { name: toolName, status: success ? 'success' : 'error' } : undefined + } +} + +function fromConversation(row: FlowConversation): Conversation { + return { + id: row.id, + title: row.title ?? undefined, + createdAt: row.created_at, + updatedAt: row.updated_at + } +} + +function assistantMessage(content: string, success: boolean, jobId: string | undefined): ChatMessage { + return { + id: `local-${randomId()}`, + role: 'assistant', + content, + success, + createdAt: now(), + jobId, + pending: false + } +} + +function finalized(messages: ChatMessage[]): ChatMessage[] { + return messages.some((m) => m.pending) + ? messages.map((m) => (m.pending ? { ...m, pending: false } : m)) + : messages +} + +function toError(e: unknown): Error { + return e instanceof Error ? e : new Error(String(e)) +} diff --git a/chat-sdk/src/config.ts b/chat-sdk/src/config.ts new file mode 100644 index 0000000000..88dc06781a --- /dev/null +++ b/chat-sdk/src/config.ts @@ -0,0 +1,81 @@ +import type { ChatOptions, FetchLike, HistoryMode, StorageLike, TokenSource } from './types' + +export interface ResolvedConfig { + flowPath: string + baseUrl: string + workspace: string + token: TokenSource | undefined + history: HistoryMode + /** The caller chose `history`; a failing server history is then an error, not a fallback. */ + historyExplicit: boolean + inputs: Record + fetch: FetchLike | undefined + storage: StorageLike | undefined + storageKey: string | undefined + pageSize: number + onFinish: ChatOptions['onFinish'] + onError: ChatOptions['onError'] +} + +export interface RawAppContext { + baseUrl: string + workspace: string + /** The viewer's SDK token in a sandboxed raw app; the session cookie otherwise. */ + token?: string +} + +/** + * The Windmill a raw app bundle runs in. A sandboxed app gets `window.process.env` + * from its wrapper once the viewer consented to the declared SDK scopes; an + * unsandboxed one runs on the Windmill origin with the viewer's session and only + * has `window.ctx`. + */ +export function detectRawApp(): RawAppContext | undefined { + const g = globalThis as { + process?: { env?: Record } + ctx?: { workspace?: unknown } + location?: { origin?: string } + } + const env = g.process?.env + if (env?.WM_RAW_APP === 'true' && env.WM_TOKEN && env.BASE_URL && env.WM_WORKSPACE) { + return { baseUrl: env.BASE_URL, workspace: env.WM_WORKSPACE, token: env.WM_TOKEN } + } + const workspace = g.ctx?.workspace + const origin = g.location?.origin + if (typeof workspace === 'string' && workspace && origin && origin !== 'null') { + return { baseUrl: origin, workspace } + } + return undefined +} + +export function resolveConfig(options: ChatOptions): ResolvedConfig { + if (!options.flowPath) throw new Error('windmill-chat: flowPath is required') + const explicitToken = options.token !== undefined + const detected = + options.baseUrl && options.workspace && explicitToken ? undefined : detectRawApp() + const baseUrl = options.baseUrl ?? detected?.baseUrl + const workspace = options.workspace ?? detected?.workspace + if (!baseUrl || !workspace) { + throw new Error( + 'windmill-chat: pass baseUrl and workspace. They are only detected inside a raw app: an unsandboxed one on the Windmill origin, or a sandboxed one whose policy declares frontend SDK scopes.' + ) + } + // The raw app's token belongs to its own Windmill; it never travels to another origin. + const token = + options.token ?? (options.baseUrl === undefined || options.baseUrl === detected?.baseUrl ? detected?.token : undefined) + return { + flowPath: options.flowPath, + baseUrl, + workspace, + token, + history: options.history ?? (explicitToken ? 'local' : 'server'), + historyExplicit: options.history !== undefined, + inputs: options.inputs ?? {}, + fetch: options.fetch, + storage: options.storage, + storageKey: options.storageKey, + pageSize: options.pageSize ?? 50, + onFinish: options.onFinish, + onError: options.onError + } +} diff --git a/chat-sdk/src/follow.ts b/chat-sdk/src/follow.ts new file mode 100644 index 0000000000..90631bbc8f --- /dev/null +++ b/chat-sdk/src/follow.ts @@ -0,0 +1,54 @@ +import type { WindmillChatApi } from './api' +import { createStreamEventParser, type AgentStreamEvent } from './stream' +import { abortError, sleep } from './utils' + +const RECONNECT_DELAY_MS = 300 + +export type FollowEvent = + /** Agent events decoded from the job's result stream; empty when a chunk ended mid-line. */ + | { type: 'stream'; events: AgentStreamEvent[] } + | { type: 'completed'; result: unknown } + +/** + * Follows a job to completion across the server's stream timeouts: every + * connection resumes from the last `stream_offset`, so no delta is repeated and + * the flow is never re-run. `onOffset` reports each offset so a caller can + * resume later from another connection (see the AI SDK transport). + */ +export async function* followJob( + api: WindmillChatApi, + jobId: string, + options: { signal?: AbortSignal; streamOffset?: number; onOffset?: (offset: number) => void } = {} +): AsyncGenerator { + const parser = createStreamEventParser() + let offset = options.streamOffset + while (true) { + let timedOut = false + for await (const update of api.streamJob(jobId, { streamOffset: offset, signal: options.signal })) { + if (update.type === 'ping') continue + if (update.type === 'timeout') { + timedOut = true + break + } + if (update.type === 'error') throw new Error(update.error) + if (update.type === 'notfound') throw new Error(`Job ${jobId} not found`) + if (update.stream_offset !== undefined) { + offset = update.stream_offset + options.onOffset?.(offset) + } + if (update.new_result_stream) { + yield { type: 'stream', events: parser.push(update.new_result_stream) } + } + if (update.completed) { + const rest = parser.flush() + if (rest.length > 0) yield { type: 'stream', events: rest } + yield { type: 'completed', result: update.only_result } + return + } + } + if (options.signal?.aborted) throw abortError() + // The server closes the connection after its timeout; a dropped connection looks + // the same minus the event. Either way the offset lets the next one resume. + if (!timedOut) await sleep(RECONNECT_DELAY_MS, options.signal) + } +} diff --git a/chat-sdk/src/history.ts b/chat-sdk/src/history.ts new file mode 100644 index 0000000000..485dd056cf --- /dev/null +++ b/chat-sdk/src/history.ts @@ -0,0 +1,90 @@ +import type { ChatMessage, Conversation, StorageLike } from './types' + +export interface LocalHistory { + listConversations(): Conversation[] + getMessages(conversationId: string): ChatMessage[] + upsertConversation(conversation: Conversation): void + saveMessages(conversationId: string, messages: ChatMessage[]): void + deleteConversation(conversationId: string): void +} + +interface Snapshot { + v: 1 + conversations: Conversation[] + messages: Record +} + +const MAX_CONVERSATIONS = 100 + +/** + * Conversation history in the browser, for a credential shared by every visitor + * (server history would show them each other's chats). One key per flow, and per + * `storageKey` when the caller sets one; every operation re-reads the store so + * several tabs stay consistent. + */ +export function createLocalHistory(storage: StorageLike | undefined, key: string): LocalHistory { + const store = storage ?? defaultStorage() + const read = (): Snapshot => { + try { + const raw = store.getItem(key) + if (raw) { + const parsed = JSON.parse(raw) as Snapshot + if (parsed && parsed.v === 1) return parsed + } + } catch { + // unreadable: start over + } + return { v: 1, conversations: [], messages: {} } + } + const write = (snapshot: Snapshot) => { + try { + store.setItem(key, JSON.stringify(snapshot)) + } catch { + // quota or private mode: the page keeps working from memory + } + } + return { + listConversations: () => read().conversations, + getMessages: (id) => read().messages[id] ?? [], + upsertConversation(conversation) { + const s = read() + const rest = s.conversations.filter((c) => c.id !== conversation.id) + s.conversations = [conversation, ...rest] + for (const dropped of s.conversations.splice(MAX_CONVERSATIONS)) { + delete s.messages[dropped.id] + } + write(s) + }, + saveMessages(id, messages) { + const s = read() + s.messages[id] = messages.map((m) => ({ ...m, pending: false })) + write(s) + }, + deleteConversation(id) { + const s = read() + s.conversations = s.conversations.filter((c) => c.id !== id) + delete s.messages[id] + write(s) + } + } +} + +function defaultStorage(): StorageLike { + try { + const ls = globalThis.localStorage + if (ls) { + const probe = '__windmill_chat_probe__' + ls.setItem(probe, '1') + ls.removeItem(probe) + return ls + } + } catch { + // no localStorage (SSR, blocked storage): fall through + } + const memory = new Map() + return { + getItem: (k) => memory.get(k) ?? null, + setItem: (k, v) => void memory.set(k, v), + removeItem: (k) => void memory.delete(k) + } +} diff --git a/chat-sdk/src/index.ts b/chat-sdk/src/index.ts new file mode 100644 index 0000000000..353c814154 --- /dev/null +++ b/chat-sdk/src/index.ts @@ -0,0 +1,29 @@ +export { createChat } from './chat' +export { detectRawApp, type RawAppContext } from './config' +export { + WindmillChatApi, + WindmillApiError, + readServerSentEvents, + type WindmillChatApiOptions, + type FlowConversation, + type FlowConversationMessage, + type JobUpdateEvent, + type CompletedJobResult +} from './api' +export { parseStreamEvents, createStreamEventParser, type AgentStreamEvent } from './stream' +export { followJob, type FollowEvent } from './follow' +export { extractChatAnswer, conversationIdFor } from './utils' +export type { + Chat, + ChatMessage, + ChatOptions, + ChatRole, + ChatState, + ChatStatus, + Conversation, + FetchLike, + HistoryMode, + StorageLike, + TokenSource, + ToolInvocation +} from './types' diff --git a/chat-sdk/src/react.ts b/chat-sdk/src/react.ts new file mode 100644 index 0000000000..48d3b1037c --- /dev/null +++ b/chat-sdk/src/react.ts @@ -0,0 +1,67 @@ +import { useEffect, useMemo, useRef, useSyncExternalStore } from 'react' +import { createChat } from './chat' +import type { Chat, ChatOptions, ChatState } from './types' + +export type UseWindmillChat = ChatState & + Pick< + Chat, + | 'sendMessage' + | 'stop' + | 'newConversation' + | 'selectConversation' + | 'loadConversations' + | 'deleteConversation' + | 'loadOlderMessages' + > & { chat: Chat } + +/** + * A chat on a chat-mode flow. The chat is created once per `flowPath`, `baseUrl`, + * `workspace`, `history`, `storageKey` and credential, and destroyed on unmount. A + * credential change is a new user, whose chat must not carry the previous one's + * state: a different token string, or a switch between no token, a token string + * and a token function, all recreate it. A token function is read through a ref + * on every call, so a new closure per render changes what the next call runs and + * nothing else; pass a `storageKey` per user when local history must not be shared. + * The callbacks and `inputs` are read the same way: the latest render's values go + * with the next message. + */ +export function useWindmillChat(options: ChatOptions): UseWindmillChat { + const latest = useRef(options) + latest.current = options + const credential = + typeof options.token === 'function' ? 'fn' : typeof options.token === 'string' ? `str:${options.token}` : 'none' + const chat = useMemo( + () => + createChat({ + ...options, + token: + typeof options.token === 'function' + ? () => { + const token = latest.current.token + return typeof token === 'function' ? token() : (token ?? '') + } + : options.token, + onFinish: (turn) => latest.current.onFinish?.(turn), + onError: (error, turn) => latest.current.onError?.(error, turn) + }), + // eslint-disable-next-line react-hooks/exhaustive-deps + [options.flowPath, options.baseUrl, options.workspace, options.history, options.storageKey, credential] + ) + useEffect(() => () => chat.destroy(), [chat]) + const state = useSyncExternalStore(chat.subscribe, chat.getState, chat.getState) + return useMemo( + () => ({ + ...state, + chat, + sendMessage: (text, options) => + chat.sendMessage(text, { ...options, inputs: { ...latest.current.inputs, ...options?.inputs } }), + stop: chat.stop, + newConversation: chat.newConversation, + selectConversation: chat.selectConversation, + loadConversations: chat.loadConversations, + deleteConversation: chat.deleteConversation, + loadOlderMessages: chat.loadOlderMessages + }), + [state, chat] + ) +} diff --git a/chat-sdk/src/stream.ts b/chat-sdk/src/stream.ts new file mode 100644 index 0000000000..c6969bb7b4 --- /dev/null +++ b/chat-sdk/src/stream.ts @@ -0,0 +1,65 @@ +/** The events an AI agent step streams, one JSON object per line of the job's result stream. */ +export type AgentStreamEvent = + | { type: 'token_delta'; content: string } + | { type: 'reasoning_token_delta'; content: string } + | { type: 'tool_call'; call_id: string; function_name: string } + | { type: 'tool_call_arguments'; call_id: string; function_name: string; arguments: string } + | { type: 'tool_execution'; call_id: string; function_name: string } + | { + type: 'tool_result' + call_id: string + function_name: string + result: string + success: boolean + } + +const KNOWN_TYPES = new Set([ + 'token_delta', + 'reasoning_token_delta', + 'tool_call', + 'tool_call_arguments', + 'tool_execution', + 'tool_result' +]) + +/** + * Incremental parser for the `new_result_stream` chunks of a job update. A chunk is + * not guaranteed to end on a line boundary, so an incomplete last line waits for the + * next `push` (or `flush` once the job completes). + */ +export function createStreamEventParser() { + let pending = '' + return { + push(chunk: string): AgentStreamEvent[] { + pending += chunk + const lastNewline = pending.lastIndexOf('\n') + if (lastNewline === -1) return [] + const complete = pending.slice(0, lastNewline) + pending = pending.slice(lastNewline + 1) + return parseStreamEvents(complete) + }, + flush(): AgentStreamEvent[] { + const rest = pending + pending = '' + return parseStreamEvents(rest) + } + } +} + +/** Parses complete NDJSON lines; lines that aren't agent events are skipped. */ +export function parseStreamEvents(ndjson: string): AgentStreamEvent[] { + const events: AgentStreamEvent[] = [] + for (const line of ndjson.split('\n')) { + const trimmed = line.trim() + if (!trimmed) continue + try { + const parsed = JSON.parse(trimmed) + if (parsed && typeof parsed === 'object' && KNOWN_TYPES.has(parsed.type)) { + events.push(parsed as AgentStreamEvent) + } + } catch { + // not an agent event + } + } + return events +} diff --git a/chat-sdk/src/types.ts b/chat-sdk/src/types.ts new file mode 100644 index 0000000000..31b44cdd8f --- /dev/null +++ b/chat-sdk/src/types.ts @@ -0,0 +1,123 @@ +export type ChatRole = 'user' | 'assistant' | 'tool' | 'system' + +/** + * - `idle`: ready for a message + * - `submitted`: the message was sent, no answer has started streaming yet + * - `streaming`: the answer is arriving + * - `error`: the last turn failed; `error` holds why. Sending again is allowed. + */ +export type ChatStatus = 'idle' | 'submitted' | 'streaming' | 'error' + +/** + * Where conversation history lives. + * - `server`: Windmill's conversation store. Each Windmill user only sees their own + * conversations, so use it with the viewer's own session or a per-user token. + * - `local`: the browser's storage. Right for a token shared by every visitor. + * - `none`: nothing is kept beyond the current page. + */ +export type HistoryMode = 'server' | 'local' | 'none' + +export interface ToolInvocation { + callId?: string + name: string + /** The arguments the model passed, as a JSON string. */ + arguments?: string + result?: string + status: 'running' | 'success' | 'error' +} + +export interface ChatMessage { + id: string + role: ChatRole + content: string + /** The model's reasoning summary, when the provider streams one. */ + reasoning?: string + /** Set on `tool` messages that came from the live stream. */ + tool?: ToolInvocation + success: boolean + createdAt: string + jobId?: string + /** The flow step that produced the message. */ + stepName?: string + /** True while the message is optimistic or still streaming. */ + pending: boolean + /** Id of the persisted row once the server has it; `id` itself never changes, so list keys stay stable. */ + serverId?: string + /** The server's cursor for a persisted message; unset for one created on the client. */ + seq?: number +} + +export interface Conversation { + id: string + title: string | undefined + createdAt: string + updatedAt: string +} + +export interface ChatState { + conversationId: string | undefined + messages: ChatMessage[] + status: ChatStatus + error: Error | undefined + conversations: Conversation[] + /** Where history is read from. Starts as configured; drops from `server` to `local` when the credential cannot read conversations. */ + history: HistoryMode + loadingMessages: boolean + hasMoreMessages: boolean +} + +export type TokenSource = string | (() => string | Promise) + +export type StorageLike = Pick + +export type FetchLike = (input: string | URL | Request, init?: RequestInit) => Promise + +export interface ChatOptions { + /** Path of a deployed flow with chat mode enabled, e.g. `f/support/assistant`. */ + flowPath: string + /** Windmill origin, e.g. `https://app.windmill.dev`. Detected inside a raw app. */ + baseUrl?: string + /** Detected inside a raw app. */ + workspace?: string + /** + * A Windmill token, or a function returning one (called before every request, so it + * can fetch a short-lived token from your backend). Omit it to use the viewer's + * session: the cookie on the Windmill origin, or a sandboxed raw app's SDK token. + */ + token?: TokenSource + /** Defaults to `server` with the viewer's session and `local` with an explicit token. */ + history?: HistoryMode + /** Extra flow inputs sent with every message, next to `user_message`. */ + inputs?: Record + fetch?: FetchLike + /** Backing store for `local` history. Defaults to `localStorage`. */ + storage?: StorageLike + /** + * Namespace for `local` history, e.g. the signed-in user's id. Local history is + * per browser, per flow; without this, users sharing a browser share it. + */ + storageKey?: string + /** Messages fetched per page of server history. */ + pageSize?: number + /** Called once a turn has its answer (a failed flow included: its error is the answer). */ + onFinish?: (turn: { conversationId: string; jobId?: string; messages: ChatMessage[] }) => void + /** Called when a turn could not run or be followed; `state.error` holds the same error. */ + onError?: (error: Error, turn: { conversationId: string; jobId?: string }) => void +} + +export interface Chat { + getState(): ChatState + /** Calls `listener` now and on every change; returns the unsubscribe function (Svelte store contract). */ + subscribe(listener: (state: ChatState) => void): () => void + /** Sends a message in the current conversation, starting one when there is none. Resolves when the answer is complete. */ + sendMessage(text: string, options?: { inputs?: Record }): Promise + /** Stops following the answer and asks Windmill to cancel the run. */ + stop(): Promise + newConversation(): void + selectConversation(conversationId: string): Promise + loadConversations(options?: { page?: number; perPage?: number }): Promise + deleteConversation(conversationId: string): Promise + loadOlderMessages(): Promise + /** Stops background work (stream, polling) and writes local history out. The chat stays usable. */ + destroy(): void +} diff --git a/chat-sdk/src/utils.ts b/chat-sdk/src/utils.ts new file mode 100644 index 0000000000..fe7e5d8960 --- /dev/null +++ b/chat-sdk/src/utils.ts @@ -0,0 +1,134 @@ +export function randomId(): string { + const c = globalThis.crypto + if (c?.randomUUID) return c.randomUUID() + // `randomUUID` needs a secure context; a plain http dev origin has `getRandomValues` only. + const bytes = new Uint8Array(16) + c.getRandomValues(bytes) + return formatUuid(bytes, 4) +} + +const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i + +export function isUuid(value: string): boolean { + return UUID_RE.test(value) +} + +/** + * The Windmill conversation id for an arbitrary chat id. A UUID is used as is; + * anything else (an AI SDK chat id, for instance) maps to the same UUID every time, + * so a page can reopen its conversation without storing a second id. Hashed in plain + * JS: `crypto.subtle` only exists in secure contexts, and the mapping must not depend + * on the origin's scheme. + */ +export function conversationIdFor(chatId: string): string { + if (isUuid(chatId)) return chatId.toLowerCase() + const bytes = new TextEncoder().encode(`windmill-chat:${chatId}`) + const out = new Uint8Array(16) + for (const [i, seed] of [0xcbf29ce484222325n, 0x84222325cbf29ce4n].entries()) { + let h = fnv1a64(bytes, seed) + for (let b = 7; b >= 0; b--) { + out[i * 8 + b] = Number(h & 0xffn) + h >>= 8n + } + } + return formatUuid(out, 5) +} + +function fnv1a64(bytes: Uint8Array, seed: bigint): bigint { + let h = seed + for (const byte of bytes) { + h ^= BigInt(byte) + h = (h * 0x100000001b3n) & 0xffffffffffffffffn + } + return h +} + +function formatUuid(bytes: Uint8Array, version: 4 | 5): string { + bytes[6] = (bytes[6] & 0x0f) | (version << 4) + bytes[8] = (bytes[8] & 0x3f) | 0x80 + const hex = Array.from(bytes, (b) => b.toString(16).padStart(2, '0')).join('') + return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}` +} + +export function parseJsonOr(text: string | undefined): unknown { + if (text === undefined) return undefined + try { + return JSON.parse(text) + } catch { + return text + } +} + +export function now(): string { + return new Date().toISOString() +} + +/** Same rule as the server: the first message, cut to 25 characters. */ +export function conversationTitle(firstMessage: string): string { + const chars = Array.from(firstMessage) + return chars.length > 25 ? `${chars.slice(0, 25).join('')}...` : firstMessage +} + +export function sleep(ms: number, signal?: AbortSignal): Promise { + return new Promise((resolve, reject) => { + if (signal?.aborted) return reject(abortError()) + const onAbort = () => { + clearTimeout(timer) + reject(abortError()) + } + const timer = setTimeout(() => { + signal?.removeEventListener('abort', onAbort) + resolve() + }, ms) + signal?.addEventListener('abort', onAbort, { once: true }) + }) +} + +export function abortError(): Error { + return new DOMException('The operation was aborted', 'AbortError') +} + +export function isAbortError(e: unknown): boolean { + return e instanceof Error && e.name === 'AbortError' +} + +/** + * The text a chat shows for a flow result, following what Windmill persists as the + * assistant message when the last step is not an AI agent: `windmill_chat_answer` + * when the result carries one (null means no message), an agent result's `output`, + * a string as is, anything else as JSON. + */ +export function extractChatAnswer(result: unknown): string | undefined { + if (result === null || result === undefined) return undefined + if (typeof result === 'string') return result + if (typeof result === 'object' && !Array.isArray(result)) { + const obj = result as Record + if ('windmill_chat_answer' in obj) return formatAnswer(obj.windmill_chat_answer) + if ('output' in obj && Array.isArray(obj.messages)) return formatAnswer(obj.output) + } + return JSON.stringify(result, null, 2) +} + +function formatAnswer(value: unknown): string | undefined { + if (value === null || value === undefined) return undefined + return typeof value === 'string' ? value : JSON.stringify(value, null, 2) +} + +/** A completed job whose result is Windmill's error envelope. */ +export function isErrorResult(result: unknown): result is { error: Record } { + return ( + typeof result === 'object' && + result !== null && + 'error' in result && + typeof (result as { error: unknown }).error === 'object' && + (result as { error: unknown }).error !== null + ) +} + +export function errorResultMessage(result: { error: Record }): string { + const { message, name } = result.error + if (typeof message === 'string' && message) { + return typeof name === 'string' && name && name !== 'Error' ? `${name}: ${message}` : message + } + return JSON.stringify(result.error, null, 2) +} diff --git a/chat-sdk/test/ai-sdk-chat.test.ts b/chat-sdk/test/ai-sdk-chat.test.ts new file mode 100644 index 0000000000..a0bbb7b8a1 --- /dev/null +++ b/chat-sdk/test/ai-sdk-chat.test.ts @@ -0,0 +1,62 @@ +import { describe, expect, test } from 'bun:test' +import { Chat } from '@ai-sdk/react' +import { createWindmillChatTransport } from '../src/ai-sdk' +import { fetchMock, json, ndjson, sse, text } from './support' + +const FLOW = 'f/chat/agent' + +/** The AI SDK's own chunk processor consuming the transport, as `useChat` would. */ +describe('AI SDK Chat over the Windmill transport', () => { + test('builds the assistant message parts and settles to ready', async () => { + let jobs = 0 + const { fetch, calls } = fetchMock( + (c) => + c.method === 'POST' && c.url.pathname === `/api/w/ws/jobs/run/f/${FLOW}` ? text(`job-${++jobs}`) : undefined, + (c) => + c.url.pathname.endsWith('/getupdate_sse/job-1') + ? sse([ + { + type: 'update', + new_result_stream: ndjson( + { type: 'tool_call', call_id: 'c1', function_name: 'lookup' }, + { type: 'tool_call_arguments', call_id: 'c1', function_name: 'lookup', arguments: '{"q":1}' }, + { type: 'tool_result', call_id: 'c1', function_name: 'lookup', result: '42', success: true }, + { type: 'reasoning_token_delta', content: 'so ' }, + { type: 'token_delta', content: 'The answer ' }, + { type: 'token_delta', content: 'is 42' } + ), + stream_offset: 6, + completed: true, + only_result: { output: 'The answer is 42', messages: [] } + } + ]) + : undefined, + (c) => + c.url.pathname.endsWith('/getupdate_sse/job-2') + ? sse([{ type: 'update', completed: true, only_result: { error: { message: 'boom' } } }]) + : undefined, + (c) => + c.url.pathname.endsWith('/completed/get_result_maybe/job-2') ? json({ completed: true, success: false }) : undefined + ) + const transport = createWindmillChatTransport({ baseUrl: 'http://wm.test', workspace: 'ws', flowPath: FLOW, token: 'tok', fetch }) + const chat = new Chat({ id: 'e2e-chat', transport }) + + await chat.sendMessage({ text: 'what is it?' }) + + expect(chat.status).toBe('ready') + expect(chat.messages.map((m) => m.role)).toEqual(['user', 'assistant']) + const parts = chat.messages[1].parts + expect(parts.map((p) => p.type)).toEqual(['dynamic-tool', 'reasoning', 'text']) + expect(parts[0]).toMatchObject({ toolName: 'lookup', toolCallId: 'c1', state: 'output-available', input: { q: 1 }, output: 42 }) + expect(parts[1]).toMatchObject({ type: 'reasoning', text: 'so ', state: 'done' }) + expect(parts[2]).toMatchObject({ type: 'text', text: 'The answer is 42', state: 'done' }) + // Both turns of the chat ran in the same Windmill conversation. + const memoryIds = calls.filter((c) => c.method === 'POST').map((c) => c.url.searchParams.get('memory_id')) + expect(memoryIds[0]).toBe(transport.conversationId('e2e-chat')) + + await chat.sendMessage({ text: 'and now fail' }) + expect(chat.status).toBe('error') + expect(chat.error?.message).toBe('boom') + expect(memoryIds.length === 1 || calls.filter((c) => c.method === 'POST')[1].url.searchParams.get('memory_id') === memoryIds[0]).toBe(true) + }) +}) diff --git a/chat-sdk/test/ai-sdk.test.ts b/chat-sdk/test/ai-sdk.test.ts new file mode 100644 index 0000000000..7f30f26edc --- /dev/null +++ b/chat-sdk/test/ai-sdk.test.ts @@ -0,0 +1,178 @@ +import { describe, expect, test } from 'bun:test' +import type { UIMessage, UIMessageChunk } from 'ai' +import { createWindmillChatTransport, toUIMessages } from '../src/ai-sdk' +import type { ChatMessage } from '../src/types' +import { fetchMock, json, ndjson, sse, text, type Route } from './support' + +const FLOW = 'f/chat/agent' +const run: Route = (c) => + c.method === 'POST' && c.url.pathname === `/api/w/ws/jobs/run/f/${FLOW}` ? text('job-1') : undefined +const streamPath = '/api/w/ws/jobs_u/getupdate_sse/job-1' + +const userMessage = (text: string): UIMessage => ({ id: 'u1', role: 'user', parts: [{ type: 'text', text }] }) + +async function collect(stream: ReadableStream): Promise { + const chunks: UIMessageChunk[] = [] + const reader = stream.getReader() + while (true) { + const { value, done } = await reader.read() + if (done) return chunks + chunks.push(value) + } +} + +describe('createWindmillChatTransport', () => { + test('maps an agent turn to AI SDK chunks and derives the conversation from the chat id', async () => { + const { fetch, calls } = fetchMock(run, (c) => + c.url.pathname === streamPath + ? sse([ + { + type: 'update', + new_result_stream: ndjson( + { type: 'reasoning_token_delta', content: 'think' }, + { type: 'token_delta', content: 'Let me ' }, + { type: 'tool_call', call_id: 'c1', function_name: 'lookup' }, + { type: 'tool_call_arguments', call_id: 'c1', function_name: 'lookup', arguments: '{"q":1}' }, + { type: 'tool_execution', call_id: 'c1', function_name: 'lookup' } + ), + stream_offset: 5 + }, + { + type: 'update', + new_result_stream: ndjson( + { type: 'tool_result', call_id: 'c1', function_name: 'lookup', result: '{"answer":42}', success: true }, + { type: 'token_delta', content: '42' } + ), + stream_offset: 7, + completed: true, + only_result: { output: '42', messages: [] } + } + ]) + : undefined + ) + const transport = createWindmillChatTransport({ + baseUrl: 'http://wm.test', + workspace: 'ws', + flowPath: FLOW, + token: 'tok', + inputs: { tone: 'kind' }, + fetch + }) + const chunks = await collect( + await transport.sendMessages({ + trigger: 'submit-message', + chatId: 'chat-abc', + messageId: undefined, + messages: [userMessage('what is it?')], + abortSignal: undefined, + body: { locale: 'fr' } + }) + ) + + const runCall = calls.find((c) => c.method === 'POST')! + expect(runCall.body).toEqual({ tone: 'kind', locale: 'fr', user_message: 'what is it?' }) + expect(runCall.url.searchParams.get('memory_id')).toBe(transport.conversationId('chat-abc')) + expect(transport.conversationId('chat-abc')).toMatch(/^[0-9a-f]{8}-[0-9a-f]{4}-5[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/) + expect(transport.conversationId('chat-abc')).toBe(transport.conversationId('chat-abc')) + expect(transport.conversationId('chat-abd')).not.toBe(transport.conversationId('chat-abc')) + expect(transport.conversationId('4D6E5C8C-2C3B-4E1A-9F31-0B2E6F1C9D10')).toBe( + '4d6e5c8c-2c3b-4e1a-9f31-0b2e6f1c9d10' + ) + + const shape = chunks.map((c) => ('delta' in c ? `${c.type}:${c.delta}` : c.type)) + expect(shape).toEqual([ + 'start', + 'reasoning-start', + 'reasoning-delta:think', + 'text-start', + 'text-delta:Let me ', + 'reasoning-end', + 'text-end', + 'tool-input-start', + 'tool-input-available', + 'tool-output-available', + 'text-start', + 'text-delta:42', + 'text-end', + 'finish' + ]) + expect(chunks.find((c) => c.type === 'tool-input-available')).toMatchObject({ + toolCallId: 'c1', + toolName: 'lookup', + input: { q: 1 }, + dynamic: true + }) + expect(chunks.find((c) => c.type === 'tool-output-available')).toMatchObject({ output: { answer: 42 } }) + // The job finished, so there is nothing to reconnect to. + expect(await transport.reconnectToStream({ chatId: 'chat-abc' })).toBeNull() + }) + + test('answers from the flow result when nothing streamed, and reports a failed flow as an error', async () => { + let turn = 0 + const { fetch } = fetchMock( + run, + (c) => { + if (c.url.pathname !== streamPath) return undefined + turn++ + return sse([ + turn === 1 + ? { type: 'update', completed: true, only_result: { windmill_chat_answer: 'From a script' } } + : { type: 'update', completed: true, only_result: { error: { name: 'ExecutionErr', message: 'boom' } } } + ]) + }, + (c) => + c.url.pathname === '/api/w/ws/jobs_u/completed/get_result_maybe/job-1' + ? json({ completed: true, success: false }) + : undefined + ) + const transport = createWindmillChatTransport({ baseUrl: 'http://wm.test', workspace: 'ws', flowPath: FLOW, fetch }) + const send = () => + transport.sendMessages({ + trigger: 'submit-message', + chatId: 'c', + messageId: undefined, + messages: [userMessage('hi')], + abortSignal: undefined + }) + const first = await collect(await send()) + expect(first.map((c) => ('delta' in c ? c.delta : c.type))).toEqual(['start', 'text-start', 'From a script', 'text-end', 'finish']) + const second = await collect(await send()) + expect(second.map((c) => c.type)).toEqual(['start', 'error']) + expect(second[1]).toMatchObject({ errorText: 'ExecutionErr: boom' }) + }) + + test('refuses attachments with a clear error', async () => { + const transport = createWindmillChatTransport({ baseUrl: 'http://wm.test', workspace: 'ws', flowPath: FLOW, fetch: fetchMock().fetch }) + await expect( + transport.sendMessages({ + trigger: 'submit-message', + chatId: 'c', + messageId: undefined, + messages: [{ id: 'u', role: 'user', parts: [{ type: 'file', mediaType: 'image/png', url: 'data:...' }] }], + abortSignal: undefined + }) + ).rejects.toThrow('attachments are not supported') + }) +}) + +describe('toUIMessages', () => { + test('folds a turn into one assistant message with reasoning, tool and text parts', () => { + const base = { success: true, createdAt: '2026-01-01T00:00:00Z', pending: false } + const messages: ChatMessage[] = [ + { ...base, id: 'u1', role: 'user', content: 'hi' }, + { ...base, id: 't1', role: 'tool', content: 'Used lookup tool', tool: { callId: 'c1', name: 'lookup', status: 'success', arguments: '{"q":1}', result: '42' } }, + { ...base, id: 'a1', role: 'assistant', content: 'The answer is 42', reasoning: 'hmm' }, + { ...base, id: 'u2', role: 'user', content: 'thanks' }, + { ...base, id: 't2', role: 'tool', content: 'Error executing lookup', success: false, tool: { name: 'lookup', status: 'error' } } + ] + const ui = toUIMessages(messages) + expect(ui.map((m) => [m.id, m.role, m.parts.map((p) => p.type)])).toEqual([ + ['u1', 'user', ['text']], + ['t1', 'assistant', ['dynamic-tool', 'reasoning', 'text']], + ['u2', 'user', ['text']], + ['t2', 'assistant', ['dynamic-tool']] + ]) + expect(ui[1].parts[0]).toMatchObject({ toolCallId: 'c1', toolName: 'lookup', state: 'output-available', input: { q: 1 }, output: 42 }) + expect(ui[3].parts[0]).toMatchObject({ state: 'output-error', errorText: 'Error executing lookup' }) + }) +}) diff --git a/chat-sdk/test/assistant-ui.test.ts b/chat-sdk/test/assistant-ui.test.ts new file mode 100644 index 0000000000..5641dc2907 --- /dev/null +++ b/chat-sdk/test/assistant-ui.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, test } from 'bun:test' +import { groupTurns, toThreadMessage } from '../src/assistant-ui' +import type { ChatMessage } from '../src/types' + +const base = { success: true, createdAt: '2026-01-01T00:00:00Z', pending: false } + +describe('assistant-ui conversion', () => { + test('groups rows into turns and renders tool calls as content parts', () => { + const messages: ChatMessage[] = [ + { ...base, id: 'u1', role: 'user', content: 'hi' }, + { ...base, id: 't1', role: 'tool', content: 'Used lookup tool', tool: { callId: 'c1', name: 'lookup', status: 'success', arguments: '{"q":1}', result: '42' } }, + { ...base, id: 'a1', role: 'assistant', content: 'The answer is 42', reasoning: 'hmm' }, + { ...base, id: 'u2', role: 'user', content: 'again' }, + { ...base, id: 'a2', role: 'assistant', content: 'partial', pending: true } + ] + const turns = groupTurns(messages) + expect(turns.map((t) => [t.id, t.role, t.messages.length])).toEqual([ + ['u1', 'user', 1], + ['t1', 'assistant', 2], + ['u2', 'user', 1], + ['a2', 'assistant', 1] + ]) + const answer = toThreadMessage(turns[1]) + expect(answer).toMatchObject({ id: 't1', role: 'assistant', status: { type: 'complete', reason: 'stop' } }) + expect(answer.content).toEqual([ + { type: 'tool-call', toolCallId: 'c1', toolName: 'lookup', args: { q: 1 }, argsText: '{"q":1}', result: 42, isError: false }, + { type: 'reasoning', text: 'hmm' }, + { type: 'text', text: 'The answer is 42' } + ]) + expect(toThreadMessage(turns[3]).status).toEqual({ type: 'running' }) + expect(toThreadMessage(turns[0])).toMatchObject({ role: 'user', content: [{ type: 'text', text: 'hi' }] }) + }) + + test('marks a failed answer as incomplete', () => { + const [turn] = groupTurns([{ ...base, id: 'a', role: 'assistant', content: 'boom', success: false }]) + expect(toThreadMessage(turn).status).toEqual({ type: 'incomplete', reason: 'error', error: 'boom' }) + }) +}) diff --git a/chat-sdk/test/chat.test.ts b/chat-sdk/test/chat.test.ts new file mode 100644 index 0000000000..93f8752dcc --- /dev/null +++ b/chat-sdk/test/chat.test.ts @@ -0,0 +1,671 @@ +import { describe, expect, test } from 'bun:test' +import { createChat } from '../src/chat' +import type { ChatOptions } from '../src/types' +import { fetchMock, json, memoryStorage, messageRow, ndjson, sse, sseTimed, text, type Route } from './support' + +const BASE = 'http://wm.test' +const FLOW = 'f/chat/agent' + +const run: Route = (c) => + c.method === 'POST' && c.url.pathname === `/api/w/ws/jobs/run/f/${FLOW}` ? text('job-1') : undefined + +const streamPath = '/api/w/ws/jobs_u/getupdate_sse/job-1' + +function options(extra: Partial, fetch: ChatOptions['fetch']): ChatOptions { + return { flowPath: FLOW, baseUrl: BASE, workspace: 'ws', fetch, storage: memoryStorage(), ...extra } +} + +describe('createChat with local history', () => { + test('streams text and tool calls, then finalizes and persists the turn', async () => { + const storage = memoryStorage() + const { fetch, calls } = fetchMock(run, (c) => + c.url.pathname === streamPath + ? sse([ + { type: 'ping' }, + { + type: 'update', + new_result_stream: ndjson( + { type: 'token_delta', content: 'Let me ' }, + { type: 'tool_call', call_id: 'c1', function_name: 'lookup' }, + { type: 'tool_call_arguments', call_id: 'c1', function_name: 'lookup', arguments: '{"q":1}' } + ), + stream_offset: 3, + flow_stream_job_id: 'agent-job' + }, + { + type: 'update', + new_result_stream: ndjson( + { type: 'tool_result', call_id: 'c1', function_name: 'lookup', result: '42', success: true }, + { type: 'token_delta', content: 'The answer is 42' } + ), + stream_offset: 5, + completed: true, + only_result: { output: 'The answer is 42', messages: [] } + } + ]) + : undefined + ) + const chat = createChat(options({ token: 'tok', storage }, fetch)) + const statuses: string[] = [] + chat.subscribe((s) => statuses.push(s.status)) + + await chat.sendMessage('what is the answer?', { inputs: { locale: 'fr' } }) + + const runCall = calls.find((c) => c.method === 'POST')! + expect(runCall.url.searchParams.get('memory_id')).toBe(chat.getState().conversationId!) + expect(runCall.body).toEqual({ locale: 'fr', user_message: 'what is the answer?' }) + expect(runCall.headers.authorization).toBe('Bearer tok') + + const state = chat.getState() + expect(state.status).toBe('idle') + expect(statuses).toContain('submitted') + expect(statuses).toContain('streaming') + expect(state.messages.map((m) => [m.role, m.content, m.pending])).toEqual([ + ['user', 'what is the answer?', false], + ['assistant', 'Let me ', false], + ['tool', 'Used lookup tool', false], + ['assistant', 'The answer is 42', false] + ]) + expect(state.messages[2].tool).toEqual({ + callId: 'c1', + name: 'lookup', + status: 'success', + arguments: '{"q":1}', + result: '42' + }) + expect(state.conversations).toHaveLength(1) + expect(state.conversations[0].title).toBe('what is the answer?') + + const reloaded = createChat(options({ token: 'tok', storage }, fetch)) + await reloaded.loadConversations() + expect(reloaded.getState().conversations.map((c) => c.id)).toEqual([state.conversationId!]) + await reloaded.selectConversation(state.conversationId!) + expect(reloaded.getState().messages.map((m) => m.content)).toEqual( + state.messages.map((m) => m.content) + ) + }) + + test('a tool call id reused by a later turn gets its own message', async () => { + const toolTurn = () => + sse([ + { + type: 'update', + new_result_stream: ndjson( + { type: 'tool_call', call_id: 'same-id', function_name: 'lookup' }, + { type: 'tool_result', call_id: 'same-id', function_name: 'lookup', result: '1', success: true }, + { type: 'token_delta', content: 'done' } + ), + stream_offset: 3, + completed: true, + only_result: { output: 'done', messages: [] } + } + ]) + const { fetch } = fetchMock(run, (c) => (c.url.pathname === streamPath ? toolTurn() : undefined)) + const chat = createChat(options({ token: 'tok' }, fetch)) + await chat.sendMessage('one') + await chat.sendMessage('two') + expect(chat.getState().messages.map((m) => m.role)).toEqual([ + 'user', + 'tool', + 'assistant', + 'user', + 'tool', + 'assistant' + ]) + }) + + test('resumes after a stream timeout from the last offset without re-running the flow', async () => { + let streamCalls = 0 + const { fetch, calls } = fetchMock(run, (c) => { + if (c.url.pathname !== streamPath) return undefined + streamCalls++ + if (streamCalls === 1) { + return sse([ + { type: 'update', new_result_stream: ndjson({ type: 'token_delta', content: 'Hel' }), stream_offset: 1 }, + { type: 'timeout' } + ]) + } + return sse([ + { + type: 'update', + new_result_stream: ndjson({ type: 'token_delta', content: 'lo' }), + stream_offset: 2, + completed: true, + only_result: 'Hello' + } + ]) + }) + const chat = createChat(options({ token: 'tok' }, fetch)) + await chat.sendMessage('hi') + + expect(calls.filter((c) => c.method === 'POST')).toHaveLength(1) + const streams = calls.filter((c) => c.url.pathname === streamPath) + expect(streams).toHaveLength(2) + expect(streams[0].url.searchParams.get('stream_offset')).toBeNull() + expect(streams[1].url.searchParams.get('stream_offset')).toBe('1') + expect(chat.getState().messages.map((m) => m.content)).toEqual(['hi', 'Hello']) + }) + + test('derives the answer from the flow result when nothing streamed', async () => { + const { fetch } = fetchMock(run, (c) => + c.url.pathname === streamPath + ? sse([{ type: 'update', completed: true, only_result: { windmill_chat_answer: 'From a script' } }]) + : undefined + ) + const chat = createChat(options({ token: 'tok' }, fetch)) + await chat.sendMessage('hi') + const [, answer] = chat.getState().messages + expect(answer.role).toBe('assistant') + expect(answer.content).toBe('From a script') + expect(answer.jobId).toBe('job-1') + }) + + test('renders a successful result that merely looks like an error envelope', async () => { + const result = { error: { message: 'domain data' } } + const { fetch } = fetchMock( + run, + (c) => (c.url.pathname === streamPath ? sse([{ type: 'update', completed: true, only_result: result }]) : undefined), + (c) => + c.url.pathname === '/api/w/ws/jobs_u/completed/get_result_maybe/job-1' + ? json({ completed: true, success: true, result }) + : undefined + ) + const chat = createChat(options({ token: 'tok' }, fetch)) + await chat.sendMessage('hi') + expect(chat.getState().messages[1]).toMatchObject({ + role: 'assistant', + success: true, + content: JSON.stringify(result, null, 2) + }) + }) + + test('reports a failed flow as an unsuccessful assistant message', async () => { + const { fetch } = fetchMock( + run, + (c) => + c.url.pathname === streamPath + ? sse([ + { + type: 'update', + completed: true, + only_result: { error: { name: 'ExecutionErr', message: 'boom' } } + } + ]) + : undefined, + (c) => + c.url.pathname === '/api/w/ws/jobs_u/completed/get_result_maybe/job-1' + ? json({ completed: true, success: false, result: { error: { message: 'boom' } } }) + : undefined + ) + const chat = createChat(options({ token: 'tok' }, fetch)) + await chat.sendMessage('hi') + const state = chat.getState() + expect(state.status).toBe('idle') + expect(state.messages[1]).toMatchObject({ role: 'assistant', success: false, content: 'ExecutionErr: boom' }) + }) +}) + +describe('createChat with server history', () => { + test('replaces the optimistic turn with the persisted rows', async () => { + const { fetch, calls } = fetchMock( + run, + (c) => + c.url.pathname === streamPath + ? sse([ + { + type: 'update', + new_result_stream: ndjson( + { type: 'reasoning_token_delta', content: 'hmm' }, + { type: 'token_delta', content: 'Hello' } + ), + stream_offset: 2, + completed: true, + only_result: { output: 'Hello', messages: [] } + } + ]) + : undefined, + (c) => + c.method === 'GET' && c.url.pathname.endsWith('/messages') + ? json([ + messageRow(11, 'user', 'hi'), + messageRow(12, 'assistant', 'Hello', { step_name: 'AI Agent', job_id: 'agent-job' }) + ]) + : undefined, + (c) => + c.url.pathname === '/api/w/ws/flow_conversations/list' + ? json([ + { + id: chatId, + workspace_id: 'ws', + flow_path: FLOW, + title: 'hi', + created_at: '2026-01-01T00:00:00Z', + updated_at: '2026-01-01T00:00:01Z', + created_by: 'admin' + } + ]) + : undefined + ) + const chat = createChat(options({}, fetch)) + let chatId = '' + const unsubscribe = chat.subscribe((s) => { + chatId = s.conversationId ?? chatId + }) + await chat.sendMessage('hi') + unsubscribe() + + const state = chat.getState() + expect(state.history).toBe('server') + expect(state.messages.map((m) => [m.serverId, m.role, m.content, m.pending])).toEqual([ + ['row-11', 'user', 'hi', false], + ['row-12', 'assistant', 'Hello', false] + ]) + // Ids stay the client's, so list keys never remount; the server id rides alongside. + expect(state.messages.map((m) => m.id.startsWith('pending-'))).toEqual([true, true]) + expect(state.messages[1]).toMatchObject({ reasoning: 'hmm', stepName: 'AI Agent', jobId: 'agent-job', seq: 12 }) + expect(state.conversations.map((c) => c.id)).toEqual([chatId]) + + const messagesCall = calls.find((c) => c.url.pathname.endsWith('/messages'))! + expect(messagesCall.url.pathname).toBe(`/api/w/ws/flow_conversations/${chatId}/messages`) + expect(messagesCall.headers.authorization).toBeUndefined() + }) + + test('keeps the streamed answer until its row lands, even when a tool row lands first', async () => { + let messageFetches = 0 + const { fetch } = fetchMock( + run, + (c) => + c.url.pathname === streamPath + ? sse([ + { + type: 'update', + new_result_stream: ndjson( + { type: 'tool_call', call_id: 'c1', function_name: 'lookup' }, + { type: 'tool_result', call_id: 'c1', function_name: 'lookup', result: '1', success: true }, + { type: 'token_delta', content: 'Final answer' } + ), + stream_offset: 3, + completed: true, + only_result: { output: 'Final answer', messages: [] } + } + ]) + : undefined, + (c) => { + if (!c.url.pathname.endsWith('/messages')) return undefined + messageFetches++ + // The assistant row is written by a task that trails the tool's. + return json( + messageFetches === 1 + ? [messageRow(21, 'user', 'hi'), messageRow(22, 'tool', 'Used lookup tool')] + : [messageRow(23, 'assistant', 'Final answer')] + ) + }, + (c) => (c.url.pathname === '/api/w/ws/flow_conversations/list' ? json([]) : undefined) + ) + const chat = createChat(options({}, fetch)) + await chat.sendMessage('hi') + expect(messageFetches).toBe(2) + expect(chat.getState().messages.map((m) => [m.serverId, m.role, m.content, m.pending])).toEqual([ + ['row-21', 'user', 'hi', false], + ['row-22', 'tool', 'Used lookup tool', false], + ['row-23', 'assistant', 'Final answer', false] + ]) + expect(chat.getState().messages[1].tool).toMatchObject({ callId: 'c1', result: '1', status: 'success' }) + }) + + test('finishes the turn from the flow result when history falls back mid-turn', async () => { + const storage = memoryStorage() + const { fetch } = fetchMock( + run, + (c) => + c.url.pathname === streamPath + ? sse([{ type: 'update', completed: true, only_result: { windmill_chat_answer: 'From a script' } }]) + : undefined, + (c) => (c.url.pathname.includes('/flow_conversations/') ? text('forbidden', 403) : undefined) + ) + const chat = createChat(options({ storage }, fetch)) + await chat.sendMessage('hi') + const state = chat.getState() + expect(state.history).toBe('local') + expect(state.status).toBe('idle') + expect(state.messages.map((m) => [m.role, m.content])).toEqual([ + ['user', 'hi'], + ['assistant', 'From a script'] + ]) + const stored = JSON.parse([...storage.data.values()][0]) + expect(stored.messages[state.conversationId!]).toHaveLength(2) + }) + + test('appends later pages of conversations', async () => { + const row = (id: string) => ({ + id, + workspace_id: 'ws', + flow_path: FLOW, + title: id, + created_at: '2026-01-01T00:00:00Z', + updated_at: '2026-01-01T00:00:00Z', + created_by: 'admin' + }) + const { fetch } = fetchMock((c) => + c.url.pathname === '/api/w/ws/flow_conversations/list' + ? json(c.url.searchParams.get('page') === '2' ? [row('c2')] : [row('c1')]) + : undefined + ) + const chat = createChat(options({}, fetch)) + await chat.loadConversations() + await chat.loadConversations({ page: 2 }) + expect(chat.getState().conversations.map((c) => c.id)).toEqual(['c1', 'c2']) + }) + + test('a turn started right after stop() is not touched by the stop sync', async () => { + let jobs = 0 + const { fetch } = fetchMock( + (c) => (c.method === 'POST' && c.url.pathname.includes('/jobs/run/f/') ? text(`job-${++jobs}`) : undefined), + (c) => + c.url.pathname.endsWith('/getupdate_sse/job-1') + ? sse([{ type: 'update', new_result_stream: ndjson({ type: 'token_delta', content: 'slow...' }), stream_offset: 1 }]) + : undefined, + (c) => + c.url.pathname.endsWith('/getupdate_sse/job-2') + ? sse([ + { + type: 'update', + new_result_stream: ndjson( + { type: 'tool_call', call_id: 'c2', function_name: 'lookup' }, + { type: 'tool_result', call_id: 'c2', function_name: 'lookup', result: '1', success: true }, + { type: 'token_delta', content: 'second' } + ), + stream_offset: 3, + completed: true, + only_result: { output: 'second', messages: [] } + } + ]) + : undefined, + (c) => (c.url.pathname.includes('/queue/cancel/') ? text('ok') : undefined), + (c) => + c.url.pathname.endsWith('/messages') + ? json([messageRow(31, 'user', 'first'), messageRow(32, 'user', 'second question'), messageRow(33, 'tool', 'Used lookup tool'), messageRow(34, 'assistant', 'second')]) + : undefined, + (c) => (c.url.pathname === '/api/w/ws/flow_conversations/list' ? json([]) : undefined) + ) + const chat = createChat(options({}, fetch)) + const first = chat.sendMessage('first') + // The stream of job-1 never completes: the connection just ends, so the turn keeps waiting. + await new Promise((r) => setTimeout(r, 50)) + const stopped = chat.stop() + await first + const second = chat.sendMessage('second question') + await stopped + await second + const roles = chat.getState().messages.map((m) => `${m.role}${m.pending ? '*' : ''}`) + expect(roles).toEqual(['user', 'assistant', 'user', 'tool', 'assistant']) + expect(chat.getState().messages.filter((m) => m.role === 'tool')).toHaveLength(1) + }) + + test('answers from the flow result when only the user row has been persisted', async () => { + let reads = 0 + const { fetch } = fetchMock( + run, + (c) => + c.url.pathname === streamPath + ? sse([{ type: 'update', completed: true, only_result: { windmill_chat_answer: 'From a script' } }]) + : undefined, + (c) => (c.url.pathname.endsWith('/messages') ? (reads++, json([messageRow(41, 'user', 'hi')])) : undefined), + (c) => (c.url.pathname === '/api/w/ws/flow_conversations/list' ? json([]) : undefined) + ) + const chat = createChat(options({}, fetch)) + await chat.sendMessage('hi') + expect(reads).toBeGreaterThan(1) + expect(chat.getState().messages.map((m) => [m.role, m.content, m.serverId])).toEqual([ + ['user', 'hi', 'row-41'], + ['assistant', 'From a script', undefined] + ]) + }) + + test('an answer the poller merged before completion is not appended again', async () => { + const { fetch } = fetchMock( + run, + (c) => + c.url.pathname === streamPath + ? sseTimed([{ type: 'update' }, 1400, { type: 'update', completed: true, only_result: { windmill_chat_answer: 'From a script' } }]) + : undefined, + (c) => + c.url.pathname.endsWith('/messages') + ? json([messageRow(51, 'user', 'hi'), messageRow(52, 'assistant', 'From a script')]) + : undefined, + (c) => (c.url.pathname === '/api/w/ws/flow_conversations/list' ? json([]) : undefined) + ) + const chat = createChat(options({}, fetch)) + await chat.sendMessage('hi') + expect(chat.getState().messages.map((m) => [m.role, m.content, m.serverId])).toEqual([ + ['user', 'hi', 'row-51'], + ['assistant', 'From a script', 'row-52'] + ]) + }) + + test('a tool row alone is not the answer of a turn that streamed no text', async () => { + let reads = 0 + const { fetch } = fetchMock( + run, + (c) => + c.url.pathname === streamPath + ? sse([{ type: 'update', completed: true, only_result: { output: 'Answer', messages: [] } }]) + : undefined, + (c) => + c.url.pathname.endsWith('/messages') + ? json(++reads === 1 ? [messageRow(61, 'user', 'hi'), messageRow(62, 'tool', 'Used lookup tool')] : [messageRow(63, 'assistant', 'Answer')]) + : undefined, + (c) => (c.url.pathname === '/api/w/ws/flow_conversations/list' ? json([]) : undefined) + ) + const chat = createChat(options({}, fetch)) + await chat.sendMessage('hi') + expect(reads).toBe(2) + expect(chat.getState().messages.map((m) => [m.role, m.content, m.serverId])).toEqual([ + ['user', 'hi', 'row-61'], + ['tool', 'Used lookup tool', 'row-62'], + ['assistant', 'Answer', 'row-63'] + ]) + }) + + test('a late answer from a stopped job is not taken as the next turn answer', async () => { + let jobs = 0 + let reads = 0 + const { fetch } = fetchMock( + (c) => (c.method === 'POST' && c.url.pathname.includes('/jobs/run/f/') ? text(`job-${++jobs}`) : undefined), + // job-1 never completes: the connection just ends, so the turn keeps waiting. + (c) => (c.url.pathname.endsWith('/getupdate_sse/job-1') ? sse([{ type: 'update' }]) : undefined), + (c) => + c.url.pathname.endsWith('/getupdate_sse/job-2') + ? sse([{ type: 'update', completed: true, only_result: { windmill_chat_answer: 'second answer' } }]) + : undefined, + // The run-only token cannot cancel: job-1 keeps running after stop(). + (c) => (c.url.pathname.includes('/queue/cancel/') ? text('forbidden', 400) : undefined), + (c) => + c.url.pathname.endsWith('/jobs_u/get/job-2') + ? json({ flow_status: { modules: [{ job: 'step-2' }] } }) + : undefined, + // Read 1 is stop()'s sync; the stopped job's answer lands after the second user row. + (c) => + c.url.pathname.endsWith('/messages') + ? json( + ++reads === 1 + ? [messageRow(71, 'user', 'first')] + : reads === 2 + ? [messageRow(72, 'user', 'second'), messageRow(73, 'assistant', 'first answer, late', { job_id: 'step-1' })] + : [messageRow(74, 'assistant', 'second answer', { job_id: 'step-2' })] + ) + : undefined, + (c) => (c.url.pathname === '/api/w/ws/flow_conversations/list' ? json([]) : undefined) + ) + const chat = createChat(options({}, fetch)) + const first = chat.sendMessage('first') + await new Promise((r) => setTimeout(r, 50)) + await chat.stop() + await first + await chat.sendMessage('second') + expect(chat.getState().messages.map((m) => [m.role, m.content, m.serverId])).toEqual([ + ['user', 'first', 'row-71'], + ['user', 'second', 'row-72'], + ['assistant', 'first answer, late', 'row-73'], + ['assistant', 'second answer', 'row-74'] + ]) + }) + + test('a failure handler answer is attributed to the turn', async () => { + let reads = 0 + const { fetch } = fetchMock( + run, + (c) => + c.url.pathname === streamPath + ? sse([{ type: 'update', completed: true, only_result: { error: { name: 'ExecutionErr', message: 'boom' } } }]) + : undefined, + (c) => + c.url.pathname.endsWith('/jobs_u/get/job-1') + ? json({ flow_status: { modules: [{ job: 'step-1' }], failure_module: { job: 'handler-1' } } }) + : undefined, + (c) => + c.url.pathname.endsWith('/messages') + ? (reads++, json([messageRow(81, 'user', 'hi'), messageRow(82, 'assistant', 'Sorry: boom', { job_id: 'handler-1', success: false })])) + : undefined, + (c) => (c.url.pathname === '/api/w/ws/flow_conversations/list' ? json([]) : undefined) + ) + const chat = createChat(options({}, fetch)) + await chat.sendMessage('hi') + expect(reads).toBe(1) + expect(chat.getState().messages.map((m) => [m.role, m.content, m.success, m.serverId])).toEqual([ + ['user', 'hi', true, 'row-81'], + ['assistant', 'Sorry: boom', false, 'row-82'] + ]) + }) + + test('deleting the current local conversation mid-turn leaves nothing behind', async () => { + const storage = memoryStorage() + const { fetch } = fetchMock(run, (c) => + c.url.pathname === streamPath + ? sse([{ type: 'update', new_result_stream: ndjson({ type: 'token_delta', content: 'partial' }), stream_offset: 1 }]) + : undefined + ) + const chat = createChat(options({ token: 'tok', storage }, fetch)) + const turn = chat.sendMessage('hello') + await new Promise((r) => setTimeout(r, 300)) + const id = chat.getState().conversationId! + await chat.deleteConversation(id) + await turn + await new Promise((r) => setTimeout(r, 400)) + expect(chat.getState().conversations).toEqual([]) + await chat.selectConversation(id) + expect(chat.getState().messages).toEqual([]) + expect([...storage.data.values()].join('')).not.toContain('hello') + }) + + test('viewing an older local conversation does not reorder history', async () => { + const storage = memoryStorage() + const { fetch } = fetchMock(run, (c) => + c.url.pathname === streamPath ? sse([{ type: 'update', completed: true, only_result: 'ok' }]) : undefined + ) + const chat = createChat(options({ token: 'tok', storage }, fetch)) + await chat.sendMessage('older') + const older = chat.getState().conversationId! + chat.newConversation() + await chat.sendMessage('newer') + const newer = chat.getState().conversationId! + await chat.selectConversation(older) + await new Promise((r) => setTimeout(r, 400)) + const again = createChat(options({ token: 'tok', storage }, fetch)) + expect((await again.loadConversations()).map((c) => c.id)).toEqual([newer, older]) + }) + + test('destroying the chat mid-turn leaves it idle', async () => { + const { fetch } = fetchMock(run, (c) => + c.url.pathname === streamPath + ? sse([{ type: 'update', new_result_stream: ndjson({ type: 'token_delta', content: 'partial' }), stream_offset: 1 }]) + : undefined + ) + const chat = createChat(options({ token: 'tok' }, fetch)) + const turn = chat.sendMessage('hello') + await new Promise((r) => setTimeout(r, 50)) + expect(chat.getState().status).toBe('streaming') + chat.destroy() + await turn + expect(chat.getState().status).toBe('idle') + expect(chat.getState().messages.every((m) => !m.pending)).toBe(true) + }) + + test('destroying the chat during a local turn keeps what it showed', async () => { + const storage = memoryStorage() + const { fetch } = fetchMock(run, (c) => + c.url.pathname === streamPath + ? sse([{ type: 'update', new_result_stream: ndjson({ type: 'token_delta', content: 'partial' }), stream_offset: 1 }]) + : undefined + ) + const chat = createChat(options({ token: 'tok', storage }, fetch)) + const turn = chat.sendMessage('hello') + await new Promise((r) => setTimeout(r, 50)) + const id = chat.getState().conversationId! + chat.destroy() + await turn + const again = createChat(options({ token: 'tok', storage }, fetch)) + await again.loadConversations() + expect(again.getState().conversations.map((c) => c.id)).toEqual([id]) + await again.selectConversation(id) + expect(again.getState().messages.map((m) => [m.role, m.content, m.pending])).toEqual([ + ['user', 'hello', false], + ['assistant', 'partial', false] + ]) + }) + + test('switching conversations keeps what a local turn showed so far', async () => { + const storage = memoryStorage() + const { fetch } = fetchMock(run, (c) => + c.url.pathname === streamPath + ? sse([{ type: 'update', new_result_stream: ndjson({ type: 'token_delta', content: 'partial' }), stream_offset: 1 }]) + : undefined + ) + const chat = createChat(options({ token: 'tok', storage }, fetch)) + const turn = chat.sendMessage('hello') + await new Promise((r) => setTimeout(r, 50)) + const id = chat.getState().conversationId! + chat.newConversation() + await turn + expect(chat.getState().messages).toEqual([]) + await chat.selectConversation(id) + expect(chat.getState().messages.map((m) => [m.role, m.content, m.pending])).toEqual([ + ['user', 'hello', false], + ['assistant', 'partial', false] + ]) + }) + + test('answers from the flow result when server history keeps failing', async () => { + const { fetch } = fetchMock( + run, + (c) => + c.url.pathname === streamPath + ? sse([{ type: 'update', completed: true, only_result: { windmill_chat_answer: 'From a script' } }]) + : undefined, + (c) => (c.url.pathname.includes('/flow_conversations/') ? text('down', 503) : undefined) + ) + const chat = createChat(options({ history: 'server' }, fetch)) + await chat.sendMessage('hi') + const state = chat.getState() + expect(state.history).toBe('server') + expect(state.status).toBe('idle') + expect(state.messages.map((m) => [m.role, m.content])).toEqual([ + ['user', 'hi'], + ['assistant', 'From a script'] + ]) + }) + + test('falls back to local history when the credential cannot read conversations', async () => { + const { fetch } = fetchMock((c) => + c.url.pathname === '/api/w/ws/flow_conversations/list' ? text('forbidden', 403) : undefined + ) + const chat = createChat(options({}, fetch)) + expect(chat.getState().history).toBe('server') + await chat.loadConversations() + expect(chat.getState().history).toBe('local') + + const explicit = createChat(options({ history: 'server' }, fetch)) + await expect(explicit.loadConversations()).rejects.toThrow('403') + expect(explicit.getState().history).toBe('server') + }) +}) diff --git a/chat-sdk/test/config.test.ts b/chat-sdk/test/config.test.ts new file mode 100644 index 0000000000..2289f20c89 --- /dev/null +++ b/chat-sdk/test/config.test.ts @@ -0,0 +1,49 @@ +import { afterEach, describe, expect, test } from 'bun:test' +import { resolveConfig } from '../src/config' + +const g = globalThis as { process?: unknown; ctx?: unknown; location?: unknown } +const originalProcess = g.process + +afterEach(() => { + g.process = originalProcess + delete g.ctx + delete g.location +}) + +describe('resolveConfig', () => { + test('explicit token defaults history to local; a session defaults to server', () => { + const base = { flowPath: 'f/a/b', baseUrl: 'http://wm.test/', workspace: 'ws' } + expect(resolveConfig({ ...base, token: 't' }).history).toBe('local') + expect(resolveConfig(base).history).toBe('server') + expect(resolveConfig({ ...base, token: 't', history: 'server' }).historyExplicit).toBe(true) + }) + + test('reads the sandboxed raw app env the wrapper injects', () => { + g.process = { + env: { WM_RAW_APP: 'true', WM_TOKEN: 'sdk-token', BASE_URL: 'http://wm.test', WM_WORKSPACE: 'ws' } + } + const config = resolveConfig({ flowPath: 'f/a/b' }) + expect(config).toMatchObject({ baseUrl: 'http://wm.test', workspace: 'ws', token: 'sdk-token', history: 'server' }) + }) + + test('keeps the raw app token off another instance', () => { + g.process = { + env: { WM_RAW_APP: 'true', WM_TOKEN: 'sdk-token', BASE_URL: 'http://wm.test', WM_WORKSPACE: 'ws' } + } + expect(resolveConfig({ flowPath: 'f/a/b', baseUrl: 'http://other.test', workspace: 'ws' }).token).toBeUndefined() + expect(resolveConfig({ flowPath: 'f/a/b', baseUrl: 'http://wm.test' }).token).toBe('sdk-token') + }) + + test('reads the unsandboxed raw app context and uses the page origin', () => { + g.ctx = { ctx: { username: 'admin' }, workspace: 'ws' } + g.location = { origin: 'http://wm.test' } + const config = resolveConfig({ flowPath: 'f/a/b' }) + expect(config).toMatchObject({ baseUrl: 'http://wm.test', workspace: 'ws', token: undefined }) + }) + + test('refuses an opaque origin without an SDK token', () => { + g.ctx = { workspace: 'ws' } + g.location = { origin: 'null' } + expect(() => resolveConfig({ flowPath: 'f/a/b' })).toThrow('frontend SDK scopes') + }) +}) diff --git a/chat-sdk/test/react.test.tsx b/chat-sdk/test/react.test.tsx new file mode 100644 index 0000000000..542d274e9e --- /dev/null +++ b/chat-sdk/test/react.test.tsx @@ -0,0 +1,80 @@ +import { GlobalRegistrator } from '@happy-dom/global-registrator' +// Test files share one process: the DOM globals must not outlive this file. +GlobalRegistrator.register() + +import { afterAll, describe, expect, test } from 'bun:test' +import React, { act } from 'react' +import { createRoot, type Root } from 'react-dom/client' +import { useWindmillChat, type UseWindmillChat } from '../src/react' +import type { ChatOptions } from '../src/types' +import { fetchMock, memoryStorage } from './support' + +afterAll(() => GlobalRegistrator.unregister()) + +const base: ChatOptions = { + flowPath: 'f/chat/agent', + baseUrl: 'http://wm.test', + workspace: 'ws', + fetch: fetchMock().fetch, + storage: memoryStorage() +} + +/** Renders the hook and hands back what it returned, rerendering with new options on demand. */ +function mountHook() { + let latest: UseWindmillChat | undefined + const Probe = (props: ChatOptions) => { + latest = useWindmillChat(props) + return null + } + const root: Root = createRoot(document.createElement('div')) + const render = (props: ChatOptions) => { + act(() => root.render()) + return latest! + } + return { render, unmount: () => act(() => root.unmount()) } +} + +describe('useWindmillChat', () => { + test('a credential change is a new chat; a new closure for the same credential is not', () => { + const { render, unmount } = mountHook() + const a = render({ ...base, token: 'user-a' }).chat + expect(render({ ...base, token: 'user-a' }).chat).toBe(a) + const b = render({ ...base, token: 'user-b' }).chat + expect(b).not.toBe(a) + + const fn1 = render({ ...base, token: () => 'fn-1' }).chat + expect(fn1).not.toBe(b) + expect(render({ ...base, token: () => 'fn-2' }).chat).toBe(fn1) + + const session = render({ ...base }).chat + expect(session).not.toBe(fn1) + const fn3 = render({ ...base, token: () => 'fn-3' }).chat + expect(fn3).not.toBe(session) + expect(render({ ...base, token: () => 'fn-3', storageKey: 'someone-else' }).chat).not.toBe(fn3) + unmount() + }) + + test('the latest inputs go with the next message', async () => { + const { fetch, calls } = fetchMock((c) => (c.method === 'POST' ? new Response('job-1') : undefined)) + const { render, unmount } = mountHook() + render({ ...base, fetch, token: 'tok', inputs: { docId: 'first' } }) + const hook = render({ ...base, fetch, token: 'tok', inputs: { docId: 'second' } }) + // The run's stream never answers here; only the request matters. + void hook.sendMessage('hi', { inputs: { extra: true } }).catch(() => {}) + await new Promise((r) => setTimeout(r, 20)) + expect(calls.find((c) => c.method === 'POST')?.body).toEqual({ docId: 'second', extra: true, user_message: 'hi' }) + unmount() + }) + + test('a token function is read through a ref, so the latest closure serves the next request', async () => { + const { fetch, calls } = fetchMock((c) => (c.url.pathname.includes('/flow_conversations/list') ? new Response('[]') : undefined)) + const { render, unmount } = mountHook() + const first = render({ ...base, fetch, history: 'server', token: () => 'first' }) + await act(() => first.loadConversations()) + const second = render({ ...base, fetch, history: 'server', token: () => 'second' }) + expect(second.chat).toBe(first.chat) + await act(() => second.loadConversations()) + expect(calls.map((c) => c.headers.authorization)).toEqual(['Bearer first', 'Bearer second']) + unmount() + }) +}) diff --git a/chat-sdk/test/stream.test.ts b/chat-sdk/test/stream.test.ts new file mode 100644 index 0000000000..84f2502133 --- /dev/null +++ b/chat-sdk/test/stream.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, test } from 'bun:test' +import { readServerSentEvents } from '../src/api' +import { createStreamEventParser, parseStreamEvents } from '../src/stream' +import { ndjson } from './support' + +describe('parseStreamEvents', () => { + test('keeps agent events and skips other lines', () => { + const events = parseStreamEvents( + ndjson( + { type: 'token_delta', content: 'Hi' }, + { type: 'reasoning_token_delta', content: 'thinking' }, + { type: 'something_else', content: 'x' }, + { type: 'tool_result', call_id: 'c1', function_name: 'lookup', result: '42', success: true } + ) + 'not json\n' + ) + expect(events.map((e) => e.type)).toEqual(['token_delta', 'reasoning_token_delta', 'tool_result']) + }) +}) + +describe('createStreamEventParser', () => { + test('holds an incomplete line until the rest arrives', () => { + const parser = createStreamEventParser() + const line = JSON.stringify({ type: 'token_delta', content: 'Hello' }) + expect(parser.push(line.slice(0, 10))).toEqual([]) + expect(parser.push(line.slice(10) + '\n' + '{"type":"token_delta",')).toEqual([ + { type: 'token_delta', content: 'Hello' } + ]) + expect(parser.push('"content":"!"}')).toEqual([]) + expect(parser.flush()).toEqual([{ type: 'token_delta', content: '!' }]) + }) +}) + +describe('readServerSentEvents', () => { + test('splits frames that straddle chunks and normalizes CRLF', async () => { + const chunks = ['data: {"a":1}\r\n\r\ndata: {"b"', ':2}\n\ndata: first\ndata: second\n\n', 'data: {"c":3}'] + const encoder = new TextEncoder() + const body = new ReadableStream({ + start(controller) { + for (const c of chunks) controller.enqueue(encoder.encode(c)) + controller.close() + } + }) + const frames: string[] = [] + for await (const data of readServerSentEvents(body)) frames.push(data) + expect(frames).toEqual(['{"a":1}', '{"b":2}', 'first\nsecond', '{"c":3}']) + }) + + test('keeps a CRLF split across chunks from ending the event', async () => { + const chunks = ['data: first\r', '\ndata: second\r\n\r\ndata: last\r'] + const encoder = new TextEncoder() + const body = new ReadableStream({ + start(controller) { + for (const c of chunks) controller.enqueue(encoder.encode(c)) + controller.close() + } + }) + const frames: string[] = [] + for await (const data of readServerSentEvents(body)) frames.push(data) + expect(frames).toEqual(['first\nsecond', 'last']) + }) +}) diff --git a/chat-sdk/test/support.ts b/chat-sdk/test/support.ts new file mode 100644 index 0000000000..c794e50f3d --- /dev/null +++ b/chat-sdk/test/support.ts @@ -0,0 +1,101 @@ +import type { FetchLike, StorageLike } from '../src/types' + +export interface RecordedCall { + method: string + url: URL + headers: Record + body: unknown +} + +export type Route = (call: RecordedCall) => Response | Promise | undefined + +/** A fetch whose responses come from the first route that answers; every call is recorded. */ +export function fetchMock(...routes: Route[]): { fetch: FetchLike; calls: RecordedCall[] } { + const calls: RecordedCall[] = [] + const fetch: FetchLike = async (input, init) => { + const url = new URL(typeof input === 'string' ? input : input instanceof URL ? input.href : input.url) + const call: RecordedCall = { + method: init?.method ?? 'GET', + url, + headers: Object.fromEntries( + Object.entries((init?.headers as Record) ?? {}).map(([k, v]) => [k.toLowerCase(), v]) + ), + body: typeof init?.body === 'string' ? JSON.parse(init.body) : undefined + } + calls.push(call) + for (const route of routes) { + const res = await route(call) + if (res) return res + } + return new Response(`no route for ${call.method} ${url.pathname}`, { status: 404 }) + } + return { fetch, calls } +} + +export function json(value: unknown, status = 200): Response { + return new Response(JSON.stringify(value), { + status, + headers: { 'content-type': 'application/json' } + }) +} + +export function text(value: string, status = 200): Response { + return new Response(value, { status }) +} + +/** A `text/event-stream` body carrying one `data:` frame per event. */ +export function sse(events: object[]): Response { + return new Response(events.map((e) => `data: ${JSON.stringify(e)}\n\n`).join(''), { + status: 200, + headers: { 'content-type': 'text/event-stream' } + }) +} + +/** Like `sse`, but a number in the list pauses that many milliseconds before the next frame. */ +export function sseTimed(events: (object | number)[]): Response { + const encoder = new TextEncoder() + const body = new ReadableStream({ + async start(controller) { + for (const e of events) { + if (typeof e === 'number') await new Promise((r) => setTimeout(r, e)) + else controller.enqueue(encoder.encode(`data: ${JSON.stringify(e)}\n\n`)) + } + controller.close() + } + }) + return new Response(body, { status: 200, headers: { 'content-type': 'text/event-stream' } }) +} + +export function ndjson(...events: object[]): string { + return events.map((e) => JSON.stringify(e)).join('\n') + '\n' +} + +export function memoryStorage(): StorageLike & { data: Map } { + const data = new Map() + return { + data, + getItem: (k) => data.get(k) ?? null, + setItem: (k, v) => void data.set(k, v), + removeItem: (k) => void data.delete(k) + } +} + +export function messageRow( + seq: number, + type: 'user' | 'assistant' | 'tool', + content: string, + extra: Record = {} +) { + return { + id: `row-${seq}`, + conversation_id: 'conv', + message_type: type, + content, + job_id: null, + created_at: '2026-01-01T00:00:00Z', + created_seq: seq, + step_name: null, + success: true, + ...extra + } +} diff --git a/chat-sdk/tsconfig.build.json b/chat-sdk/tsconfig.build.json new file mode 100644 index 0000000000..b8fc4fa324 --- /dev/null +++ b/chat-sdk/tsconfig.build.json @@ -0,0 +1,12 @@ +{ + "extends": "./tsconfig.json", + "include": ["src/**/*"], + "compilerOptions": { + "types": [], + "noEmit": false, + "declaration": true, + "emitDeclarationOnly": true, + "outDir": "dist", + "rootDir": "src" + } +} diff --git a/chat-sdk/tsconfig.json b/chat-sdk/tsconfig.json new file mode 100644 index 0000000000..fbfee1f0d5 --- /dev/null +++ b/chat-sdk/tsconfig.json @@ -0,0 +1,15 @@ +{ + "include": ["src/**/*", "test/**/*"], + "compilerOptions": { + "target": "ES2020", + "lib": ["ES2020", "DOM", "DOM.Iterable"], + "module": "ESNext", + "moduleResolution": "bundler", + "jsx": "react-jsx", + "types": ["bun"], + "strict": true, + "noEmit": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true + } +} diff --git a/chat-sdk/tsdown.config.ts b/chat-sdk/tsdown.config.ts new file mode 100644 index 0000000000..98b23c536e --- /dev/null +++ b/chat-sdk/tsdown.config.ts @@ -0,0 +1,9 @@ +import { defineConfig } from 'tsdown' + +export default defineConfig({ + entry: ['src/index.ts', 'src/react.ts', 'src/ai-sdk.ts', 'src/assistant-ui.ts'], + format: ['esm', 'cjs'], + dts: false, + external: ['react', 'ai', '@assistant-ui/react'], + target: 'es2020', +}) diff --git a/cli/src/guidance/skills.gen.ts b/cli/src/guidance/skills.gen.ts index 3f63574b83..f646b08f8c 100644 --- a/cli/src/guidance/skills.gen.ts +++ b/cli/src/guidance/skills.gen.ts @@ -5907,6 +5907,19 @@ An app can be demoed by recording a session: every interaction becomes a step ca Apply it to customer data, internal notes and anything else a viewer of the demo should not see. It costs nothing when the app is never recorded. +### Chat UIs over a flow in chat mode + +A flow deployed with chat mode on is a chat backend (streaming answer, tool calls, memory, conversation history). Do not drive it through a runnable: add \`windmill-chat\` to \`package.json\` and use it directly, it detects the app's Windmill and credential. + +\`\`\`tsx +import { useWindmillChat } from 'windmill-chat/react' + +const chat = useWindmillChat({ flowPath: 'f/support/assistant' }) +// chat.messages ({ role, content, pending, success, tool? }), chat.status, chat.sendMessage(text), chat.stop() +\`\`\` + +\`windmill-chat/ai-sdk\` gives a \`ChatTransport\` for the Vercel AI SDK's \`useChat\`, \`windmill-chat/assistant-ui\` a runtime for assistant-ui. The flow must be deployed, not a draft. A sandboxed app needs \`jobs:run\` in its frontend SDK scopes, plus \`flow_conversations:write\` for the conversation sidebar; without them the chat keeps history in the browser. + ## Backend runnables Each runnable has a unique key (used to call it from the frontend) and one of four types: @@ -6035,6 +6048,7 @@ def main(user_id: str): 6. **Mark sensitive UI with \`data-wm-no-record\`** — it is what keeps that data out of a recorded demo; passwords are handled for you. 7. **Reach for \`backendAsync\` + \`waitJob\`** for long work — never a hand-written job-polling runnable. 8. **Deploy what a path runnable points at** — a path runnable aimed at a draft fails at runtime; tell the user what needs deploying. +9. **Use \`windmill-chat\` for a chat over a chat-mode flow** — never a runnable that runs the flow and polls its stream. `, "triggers": `--- name: triggers diff --git a/frontend/src/lib/components/raw_apps/sdkScopes.ts b/frontend/src/lib/components/raw_apps/sdkScopes.ts index f0ece00c3e..b0177bb660 100644 --- a/frontend/src/lib/components/raw_apps/sdkScopes.ts +++ b/frontend/src/lib/components/raw_apps/sdkScopes.ts @@ -28,6 +28,16 @@ export const FRONTEND_SDK_SCOPES: { value: string; label: string; description: s value: 'variables:read', label: 'Read variables', description: 'Read variable values the viewer can access' + }, + { + value: 'flow_conversations:read', + label: 'Read your flow chats', + description: 'List your chat conversations with flows and read their messages' + }, + { + value: 'flow_conversations:write', + label: 'Manage your flow chats', + description: 'Read and delete your chat conversations with flows' } ] @@ -72,5 +82,5 @@ export function storeSdkConsent( ): void { try { localStorage.setItem(sdkConsentKey(viewer, workspace, path), JSON.stringify(scopes)) - } catch (_) { } + } catch (_) {} } diff --git a/frontend/src/lib/mcpEndpointTools.ts b/frontend/src/lib/mcpEndpointTools.ts index 1398e8dfa9..a9e2b9f3b2 100644 --- a/frontend/src/lib/mcpEndpointTools.ts +++ b/frontend/src/lib/mcpEndpointTools.ts @@ -1275,10 +1275,12 @@ export const mcpEndpointTools: EndpointTool[] = [ "description": "Who may open the app, and who its runnables execute as. Optional, and what omitting it means depends on the operation: creating an app defaults it to `publisher` (runs on behalf of the app's publisher and requires an authenticated viewer), while updating one keeps the mode the app is already deployed under. Neither `anonymous`, which makes the app publicly executable, nor `guest`, which opens it to anyone the identity provider authenticates, is ever assumed. A guest is only admitted where the workspace also has `guest_access_enabled`, which is checked when the session is minted and again on every guest request. Possible values: viewer, publisher, guest, anonymous" }, "on_behalf_of": { - "type": "string" + "type": "string", + "description": "The user or group the app runs as in anonymous or publisher mode (e.g. 'u/admin' or 'g/mygroup'). The authority for the app's identity." }, "on_behalf_of_email": { - "type": "string" + "type": "string", + "description": "Address of `on_behalf_of`, written through from it on every save and returned as stored. Optional; when absent it is derived from `on_behalf_of`. Sending it is optional too; it must name the same account as `on_behalf_of`, and a pair that disagrees is rejected." }, "sandbox": { "type": "boolean", @@ -1289,7 +1291,7 @@ export const mcpEndpointTools: EndpointTool[] = [ "items": { "type": "string" }, - "description": "Raw apps: author-declared scopes for the frontend SDK token. Takes effect only when `sandbox` is also true \u2014 an unsandboxed bundle runs with the viewer's own session, so no token is advertised or minted for it and this list stays inert. On a sandboxed app a non-empty list lets viewers mint (after consenting) a short-lived token carrying their own identity restricted to these scopes, handed to the app bundle so `windmill-client` calls run as the viewer. Must be a subset of the server's curated allowlist (jobs:run, jobs:read, users:read, resources:read, variables:read).\n" + "description": "Raw apps: author-declared scopes for the frontend SDK token. Takes effect only when `sandbox` is also true \u2014 an unsandboxed bundle runs with the viewer's own session, so no token is advertised or minted for it and this list stays inert. On a sandboxed app a non-empty list lets viewers mint (after consenting) a short-lived token carrying their own identity restricted to these scopes, handed to the app bundle so `windmill-client` calls run as the viewer. Must be a subset of the server's curated allowlist (jobs:run, jobs:read, users:read, resources:read, variables:read, flow_conversations:read, flow_conversations:write).\n" } } } @@ -1390,10 +1392,12 @@ export const mcpEndpointTools: EndpointTool[] = [ "description": "Who may open the app, and who its runnables execute as. Optional, and what omitting it means depends on the operation: creating an app defaults it to `publisher` (runs on behalf of the app's publisher and requires an authenticated viewer), while updating one keeps the mode the app is already deployed under. Neither `anonymous`, which makes the app publicly executable, nor `guest`, which opens it to anyone the identity provider authenticates, is ever assumed. A guest is only admitted where the workspace also has `guest_access_enabled`, which is checked when the session is minted and again on every guest request. Possible values: viewer, publisher, guest, anonymous" }, "on_behalf_of": { - "type": "string" + "type": "string", + "description": "The user or group the app runs as in anonymous or publisher mode (e.g. 'u/admin' or 'g/mygroup'). The authority for the app's identity." }, "on_behalf_of_email": { - "type": "string" + "type": "string", + "description": "Address of `on_behalf_of`, written through from it on every save and returned as stored. Optional; when absent it is derived from `on_behalf_of`. Sending it is optional too; it must name the same account as `on_behalf_of`, and a pair that disagrees is rejected." }, "sandbox": { "type": "boolean", @@ -1404,7 +1408,7 @@ export const mcpEndpointTools: EndpointTool[] = [ "items": { "type": "string" }, - "description": "Raw apps: author-declared scopes for the frontend SDK token. Takes effect only when `sandbox` is also true \u2014 an unsandboxed bundle runs with the viewer's own session, so no token is advertised or minted for it and this list stays inert. On a sandboxed app a non-empty list lets viewers mint (after consenting) a short-lived token carrying their own identity restricted to these scopes, handed to the app bundle so `windmill-client` calls run as the viewer. Must be a subset of the server's curated allowlist (jobs:run, jobs:read, users:read, resources:read, variables:read).\n" + "description": "Raw apps: author-declared scopes for the frontend SDK token. Takes effect only when `sandbox` is also true \u2014 an unsandboxed bundle runs with the viewer's own session, so no token is advertised or minted for it and this list stays inert. On a sandboxed app a non-empty list lets viewers mint (after consenting) a short-lived token carrying their own identity restricted to these scopes, handed to the app bundle so `windmill-client` calls run as the viewer. Must be a subset of the server's curated allowlist (jobs:run, jobs:read, users:read, resources:read, variables:read, flow_conversations:read, flow_conversations:write).\n" } } }, diff --git a/system_prompts/auto-generated/prompts.ts b/system_prompts/auto-generated/prompts.ts index 216c72c128..3529c99041 100644 --- a/system_prompts/auto-generated/prompts.ts +++ b/system_prompts/auto-generated/prompts.ts @@ -815,6 +815,19 @@ An app can be demoed by recording a session: every interaction becomes a step ca Apply it to customer data, internal notes and anything else a viewer of the demo should not see. It costs nothing when the app is never recorded. +### Chat UIs over a flow in chat mode + +A flow deployed with chat mode on is a chat backend (streaming answer, tool calls, memory, conversation history). Do not drive it through a runnable: add \`windmill-chat\` to \`package.json\` and use it directly, it detects the app's Windmill and credential. + +\`\`\`tsx +import { useWindmillChat } from 'windmill-chat/react' + +const chat = useWindmillChat({ flowPath: 'f/support/assistant' }) +// chat.messages ({ role, content, pending, success, tool? }), chat.status, chat.sendMessage(text), chat.stop() +\`\`\` + +\`windmill-chat/ai-sdk\` gives a \`ChatTransport\` for the Vercel AI SDK's \`useChat\`, \`windmill-chat/assistant-ui\` a runtime for assistant-ui. The flow must be deployed, not a draft. A sandboxed app needs \`jobs:run\` in its frontend SDK scopes, plus \`flow_conversations:write\` for the conversation sidebar; without them the chat keeps history in the browser. + ## Backend runnables Each runnable has a unique key (used to call it from the frontend) and one of four types: @@ -943,6 +956,7 @@ def main(user_id: str): 6. **Mark sensitive UI with \`data-wm-no-record\`** — it is what keeps that data out of a recorded demo; passwords are handled for you. 7. **Reach for \`backendAsync\` + \`waitJob\`** for long work — never a hand-written job-polling runnable. 8. **Deploy what a path runnable points at** — a path runnable aimed at a draft fails at runtime; tell the user what needs deploying. +9. **Use \`windmill-chat\` for a chat over a chat-mode flow** — never a runnable that runs the flow and polls its stream. `; export const PIPELINE_BASE = `# Data pipeline authoring diff --git a/system_prompts/auto-generated/skills/raw-app/SKILL.md b/system_prompts/auto-generated/skills/raw-app/SKILL.md index 05258118af..da4a9729be 100644 --- a/system_prompts/auto-generated/skills/raw-app/SKILL.md +++ b/system_prompts/auto-generated/skills/raw-app/SKILL.md @@ -330,6 +330,19 @@ An app can be demoed by recording a session: every interaction becomes a step ca Apply it to customer data, internal notes and anything else a viewer of the demo should not see. It costs nothing when the app is never recorded. +### Chat UIs over a flow in chat mode + +A flow deployed with chat mode on is a chat backend (streaming answer, tool calls, memory, conversation history). Do not drive it through a runnable: add `windmill-chat` to `package.json` and use it directly, it detects the app's Windmill and credential. + +```tsx +import { useWindmillChat } from 'windmill-chat/react' + +const chat = useWindmillChat({ flowPath: 'f/support/assistant' }) +// chat.messages ({ role, content, pending, success, tool? }), chat.status, chat.sendMessage(text), chat.stop() +``` + +`windmill-chat/ai-sdk` gives a `ChatTransport` for the Vercel AI SDK's `useChat`, `windmill-chat/assistant-ui` a runtime for assistant-ui. The flow must be deployed, not a draft. A sandboxed app needs `jobs:run` in its frontend SDK scopes, plus `flow_conversations:write` for the conversation sidebar; without them the chat keeps history in the browser. + ## Backend runnables Each runnable has a unique key (used to call it from the frontend) and one of four types: @@ -458,3 +471,4 @@ def main(user_id: str): 6. **Mark sensitive UI with `data-wm-no-record`** — it is what keeps that data out of a recorded demo; passwords are handled for you. 7. **Reach for `backendAsync` + `waitJob`** for long work — never a hand-written job-polling runnable. 8. **Deploy what a path runnable points at** — a path runnable aimed at a draft fails at runtime; tell the user what needs deploying. +9. **Use `windmill-chat` for a chat over a chat-mode flow** — never a runnable that runs the flow and polls its stream. diff --git a/system_prompts/base/raw-app.md b/system_prompts/base/raw-app.md index fde0b835ca..3849a465a0 100644 --- a/system_prompts/base/raw-app.md +++ b/system_prompts/base/raw-app.md @@ -95,6 +95,19 @@ An app can be demoed by recording a session: every interaction becomes a step ca Apply it to customer data, internal notes and anything else a viewer of the demo should not see. It costs nothing when the app is never recorded. +### Chat UIs over a flow in chat mode + +A flow deployed with chat mode on is a chat backend (streaming answer, tool calls, memory, conversation history). Do not drive it through a runnable: add `windmill-chat` to `package.json` and use it directly, it detects the app's Windmill and credential. + +```tsx +import { useWindmillChat } from 'windmill-chat/react' + +const chat = useWindmillChat({ flowPath: 'f/support/assistant' }) +// chat.messages ({ role, content, pending, success, tool? }), chat.status, chat.sendMessage(text), chat.stop() +``` + +`windmill-chat/ai-sdk` gives a `ChatTransport` for the Vercel AI SDK's `useChat`, `windmill-chat/assistant-ui` a runtime for assistant-ui. The flow must be deployed, not a draft. A sandboxed app needs `jobs:run` in its frontend SDK scopes, plus `flow_conversations:write` for the conversation sidebar; without them the chat keeps history in the browser. + ## Backend runnables Each runnable has a unique key (used to call it from the frontend) and one of four types: @@ -223,3 +236,4 @@ def main(user_id: str): 6. **Mark sensitive UI with `data-wm-no-record`** — it is what keeps that data out of a recorded demo; passwords are handled for you. 7. **Reach for `backendAsync` + `waitJob`** for long work — never a hand-written job-polling runnable. 8. **Deploy what a path runnable points at** — a path runnable aimed at a draft fails at runtime; tell the user what needs deploying. +9. **Use `windmill-chat` for a chat over a chat-mode flow** — never a runnable that runs the flow and polls its stream. From 57a134e2de18e27b3c1d3066a60b892af1089b30 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 14 Sep 2026 22:46:26 +0200 Subject: [PATCH 116/144] feat(ai-sessions): share session artifacts with the workspace by link (#11115) * feat(ai-sessions): share session artifacts with the workspace by link Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Pyjp67oR269QAx3b4yf4oH * chore: cache the shared artifact queries for offline sqlx Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Pyjp67oR269QAx3b4yf4oH * fix: replace a literal NUL byte in the shared artifact body limit comment Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Pyjp67oR269QAx3b4yf4oH * test: pin that a shared artifact is confined to its workspace's path Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01Pyjp67oR269QAx3b4yf4oH * fix: sanitize shared artifact markdown and validate the artifact id on every route The shared page renders another member's markdown, so ArtifactBody now runs the repo's rehype-raw + rehype-sanitize chain with the chat's link renderer on top; only the session viewer opts into the chat code block (mermaid, apply button). The link renderer keeps a link's text when its href is empty or unsafe, and the scheme check moves to a tested helper. The status route checks artifact_id like share does, so a NUL is a 400 rather than a 500. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01Pyjp67oR269QAx3b4yf4oH * fix(ai-sessions): say which way re-sharing moves an artifact link The popover offered "Update to v1" when a v2 link was open on a pinned v1, which reads as if v1 were newer. Each direction now has its own sentence and action: a newer version on screen updates the link, an older one shares that version instead, a rename updates the name. Co-Authored-By: Claude Fable 5.1 --------- Co-authored-by: Claude Opus 5 (1M context) --- ...a6cf0148a9e10c1c4ad0f99312b8e99ec1b8a.json | 41 +++ ...74b8c5beb1c3992f6413d7bcabc8159c0f9bb.json | 14 + ...9207306fb19756ab9c3a3d7824a15542a5b42.json | 66 ++++ ...804306fe2083928e57458ba936d6afde53b57.json | 55 +++ ...107739baa9ee276a99f481335fec8099f4d51.json | 25 ++ ...20260914135805_ai_shared_artifact.down.sql | 1 + .../20260914135805_ai_shared_artifact.up.sql | 32 ++ backend/src/monitor.rs | 11 + backend/summarized_schema.txt | 2 + backend/tests/ai_shared_artifacts.rs | 184 ++++++++++ .../tests/fixtures/ai_shared_artifacts.sql | 17 + backend/windmill-api/openapi.yaml | 158 +++++++++ backend/windmill-api/src/ai.rs | 4 + .../windmill-api/src/ai_shared_artifacts.rs | 322 ++++++++++++++++++ backend/windmill-api/src/lib.rs | 1 + .../windmill-common/src/global_settings.rs | 1 + backend/windmill-common/src/lib.rs | 17 + .../copilot/chat/LinkRenderer.svelte | 12 +- .../chat/artifacts/ArtifactBody.svelte | 46 +++ .../artifacts/ArtifactExportButton.svelte | 42 +++ .../chat/artifacts/ArtifactShareButton.svelte | 196 +++++++++++ .../chat/artifacts/ArtifactViewer.svelte | 83 ++--- .../chat/artifacts/artifactSharing.test.ts | 21 ++ .../copilot/chat/artifacts/artifactSharing.ts | 42 +++ .../components/copilot/chat/safeHref.test.ts | 33 ++ .../lib/components/copilot/chat/safeHref.ts | 17 + .../shared_artifacts/[id]/+page.svelte | 144 ++++++++ .../(logged)/shared_artifacts/[id]/+page.ts | 5 + 28 files changed, 1527 insertions(+), 65 deletions(-) create mode 100644 backend/.sqlx/query-0589cb0f96e17ecadae4923be70a6cf0148a9e10c1c4ad0f99312b8e99ec1b8a.json create mode 100644 backend/.sqlx/query-6774bc0ec8ca8c6c48e8e111ab074b8c5beb1c3992f6413d7bcabc8159c0f9bb.json create mode 100644 backend/.sqlx/query-945230149990abda67fdf4779529207306fb19756ab9c3a3d7824a15542a5b42.json create mode 100644 backend/.sqlx/query-d2861932a739887785658cdf89a804306fe2083928e57458ba936d6afde53b57.json create mode 100644 backend/.sqlx/query-e50660f58274e9c135ace356ea8107739baa9ee276a99f481335fec8099f4d51.json create mode 100644 backend/migrations/20260914135805_ai_shared_artifact.down.sql create mode 100644 backend/migrations/20260914135805_ai_shared_artifact.up.sql create mode 100644 backend/tests/ai_shared_artifacts.rs create mode 100644 backend/tests/fixtures/ai_shared_artifacts.sql create mode 100644 backend/windmill-api/src/ai_shared_artifacts.rs create mode 100644 frontend/src/lib/components/copilot/chat/artifacts/ArtifactBody.svelte create mode 100644 frontend/src/lib/components/copilot/chat/artifacts/ArtifactExportButton.svelte create mode 100644 frontend/src/lib/components/copilot/chat/artifacts/ArtifactShareButton.svelte create mode 100644 frontend/src/lib/components/copilot/chat/artifacts/artifactSharing.test.ts create mode 100644 frontend/src/lib/components/copilot/chat/artifacts/artifactSharing.ts create mode 100644 frontend/src/lib/components/copilot/chat/safeHref.test.ts create mode 100644 frontend/src/lib/components/copilot/chat/safeHref.ts create mode 100644 frontend/src/routes/(root)/(logged)/shared_artifacts/[id]/+page.svelte create mode 100644 frontend/src/routes/(root)/(logged)/shared_artifacts/[id]/+page.ts diff --git a/backend/.sqlx/query-0589cb0f96e17ecadae4923be70a6cf0148a9e10c1c4ad0f99312b8e99ec1b8a.json b/backend/.sqlx/query-0589cb0f96e17ecadae4923be70a6cf0148a9e10c1c4ad0f99312b8e99ec1b8a.json new file mode 100644 index 0000000000..6558ea16e5 --- /dev/null +++ b/backend/.sqlx/query-0589cb0f96e17ecadae4923be70a6cf0148a9e10c1c4ad0f99312b8e99ec1b8a.json @@ -0,0 +1,41 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO ai_shared_artifact\n (workspace_id, artifact_id, email, created_by, name, kind, version, content)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8)\n ON CONFLICT (workspace_id, email, artifact_id) DO UPDATE\n SET created_by = EXCLUDED.created_by,\n name = EXCLUDED.name,\n kind = EXCLUDED.kind,\n version = EXCLUDED.version,\n content = EXCLUDED.content,\n shared_at = now()\n RETURNING id, shared_at, (xmax = 0) AS \"inserted!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "shared_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 2, + "name": "inserted!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Int4", + "Text" + ] + }, + "nullable": [ + false, + false, + null + ] + }, + "hash": "0589cb0f96e17ecadae4923be70a6cf0148a9e10c1c4ad0f99312b8e99ec1b8a" +} diff --git a/backend/.sqlx/query-6774bc0ec8ca8c6c48e8e111ab074b8c5beb1c3992f6413d7bcabc8159c0f9bb.json b/backend/.sqlx/query-6774bc0ec8ca8c6c48e8e111ab074b8c5beb1c3992f6413d7bcabc8159c0f9bb.json new file mode 100644 index 0000000000..8dd7954460 --- /dev/null +++ b/backend/.sqlx/query-6774bc0ec8ca8c6c48e8e111ab074b8c5beb1c3992f6413d7bcabc8159c0f9bb.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM ai_shared_artifact\n WHERE shared_at <= now() - ($1::bigint::text || ' s')::interval", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "6774bc0ec8ca8c6c48e8e111ab074b8c5beb1c3992f6413d7bcabc8159c0f9bb" +} diff --git a/backend/.sqlx/query-945230149990abda67fdf4779529207306fb19756ab9c3a3d7824a15542a5b42.json b/backend/.sqlx/query-945230149990abda67fdf4779529207306fb19756ab9c3a3d7824a15542a5b42.json new file mode 100644 index 0000000000..542d523f53 --- /dev/null +++ b/backend/.sqlx/query-945230149990abda67fdf4779529207306fb19756ab9c3a3d7824a15542a5b42.json @@ -0,0 +1,66 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, email, name, kind, version, created_by, content, shared_at\n FROM ai_shared_artifact\n WHERE workspace_id = $1 AND id = $2\n AND shared_at > now() - ($3::bigint::text || ' s')::interval", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "kind", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "version", + "type_info": "Int4" + }, + { + "ordinal": 5, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 6, + "name": "content", + "type_info": "Text" + }, + { + "ordinal": 7, + "name": "shared_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Uuid", + "Int8" + ] + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false + ] + }, + "hash": "945230149990abda67fdf4779529207306fb19756ab9c3a3d7824a15542a5b42" +} diff --git a/backend/.sqlx/query-d2861932a739887785658cdf89a804306fe2083928e57458ba936d6afde53b57.json b/backend/.sqlx/query-d2861932a739887785658cdf89a804306fe2083928e57458ba936d6afde53b57.json new file mode 100644 index 0000000000..1e56d65715 --- /dev/null +++ b/backend/.sqlx/query-d2861932a739887785658cdf89a804306fe2083928e57458ba936d6afde53b57.json @@ -0,0 +1,55 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, name, kind, version, created_by, shared_at FROM ai_shared_artifact\n WHERE workspace_id = $1 AND email = $2 AND artifact_id = $3\n AND shared_at > now() - ($4::bigint::text || ' s')::interval", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "kind", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "version", + "type_info": "Int4" + }, + { + "ordinal": 4, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "shared_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Int8" + ] + }, + "nullable": [ + false, + false, + false, + false, + false, + false + ] + }, + "hash": "d2861932a739887785658cdf89a804306fe2083928e57458ba936d6afde53b57" +} diff --git a/backend/.sqlx/query-e50660f58274e9c135ace356ea8107739baa9ee276a99f481335fec8099f4d51.json b/backend/.sqlx/query-e50660f58274e9c135ace356ea8107739baa9ee276a99f481335fec8099f4d51.json new file mode 100644 index 0000000000..3068e4fe35 --- /dev/null +++ b/backend/.sqlx/query-e50660f58274e9c135ace356ea8107739baa9ee276a99f481335fec8099f4d51.json @@ -0,0 +1,25 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM ai_shared_artifact\n WHERE workspace_id = $1 AND id = $2 AND (email = $3 OR $4::bool)\n RETURNING name", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Uuid", + "Text", + "Bool" + ] + }, + "nullable": [ + false + ] + }, + "hash": "e50660f58274e9c135ace356ea8107739baa9ee276a99f481335fec8099f4d51" +} diff --git a/backend/migrations/20260914135805_ai_shared_artifact.down.sql b/backend/migrations/20260914135805_ai_shared_artifact.down.sql new file mode 100644 index 0000000000..04fa92db6c --- /dev/null +++ b/backend/migrations/20260914135805_ai_shared_artifact.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS ai_shared_artifact; diff --git a/backend/migrations/20260914135805_ai_shared_artifact.up.sql b/backend/migrations/20260914135805_ai_shared_artifact.up.sql new file mode 100644 index 0000000000..6daa3e5daf --- /dev/null +++ b/backend/migrations/20260914135805_ai_shared_artifact.up.sql @@ -0,0 +1,32 @@ +-- A copy of an AI session artifact that its author explicitly shared with the workspace. +-- Artifacts otherwise live only in the author's browser; this row exists only while the +-- share does, and the monitor deletes it once `shared_at` falls outside +-- AI_SHARED_ARTIFACT_RETENTION_SECS. +CREATE TABLE ai_shared_artifact ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE, + -- The browser-side artifact id. Unique per author so sharing the same artifact again + -- moves its one link forward rather than minting a second one. + artifact_id VARCHAR(255) NOT NULL, + email VARCHAR(255) NOT NULL, + created_by VARCHAR(255) NOT NULL, + name VARCHAR(255) NOT NULL, + kind VARCHAR(10) NOT NULL CHECK (kind IN ('md', 'html')), + version INTEGER NOT NULL, + content TEXT NOT NULL, + -- Reset on every re-share: retention counts from the last time the author shared it. + shared_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE (workspace_id, email, artifact_id) +); + +CREATE INDEX idx_ai_shared_artifact_shared_at ON ai_shared_artifact (shared_at); + +GRANT ALL ON ai_shared_artifact TO windmill_admin; +GRANT ALL ON ai_shared_artifact TO windmill_user; + +-- The handlers go through the raw pool and scope every query to the workspace themselves. +-- An admin-only policy is the backstop for a future query that reaches this table through +-- UserDB. +ALTER TABLE ai_shared_artifact ENABLE ROW LEVEL SECURITY; + +CREATE POLICY admin_policy ON ai_shared_artifact FOR ALL TO windmill_admin USING (true); diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index d18b23e1c0..493abd172e 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -1912,6 +1912,17 @@ pub async fn delete_expired_items(db: &DB) -> () { tracing::info!("deleted {} expired otel trace spans", deleted_spans); } + if let Err(e) = sqlx::query!( + "DELETE FROM ai_shared_artifact + WHERE shared_at <= now() - ($1::bigint::text || ' s')::interval", + windmill_common::ai_shared_artifact_retention_secs(), + ) + .execute(db) + .await + { + tracing::error!("Error deleting expired shared AI artifacts: {:?}", e); + } + let audit_retention_days = audit_log_retention_days().await; let audit_retention_secs: i64 = audit_retention_days * 60 * 60 * 24; diff --git a/backend/summarized_schema.txt b/backend/summarized_schema.txt index 41dd70ca93..1ab05bc584 100644 --- a/backend/summarized_schema.txt +++ b/backend/summarized_schema.txt @@ -40,6 +40,8 @@ agent_token_blacklist: token(char), expires_at(ts), blacklisted_at(ts), blacklis ai_agent_memory: workspace_id(char), conversation_id(uuid), step_id(char), messages(jsonb), created_at(ts), updated_at(ts) ai_free_token_daily_usage: day(date), cost_nanos(bigint), updated_at(ts) ai_free_token_usage: email(char), cost_nanos(bigint), updated_at(ts) +ai_shared_artifact: id(uuid), workspace_id(char), artifact_id(char), email(char), created_by(char), name(char), kind(char), version(int), content(text), shared_at(ts) + FK: (workspace_id) -> workspace(id) ai_token_usage: workspace_id(char), day(date), email(char), provider(char), model(char), session_id(char), input_tokens(bigint), cache_read_tokens(bigint), cache_write_tokens(bigint), output_tokens(bigint), reported_cost_nano_usd(bigint), requests(bigint), updated_at(ts) FK: (workspace_id) -> workspace(id) alerts: id(int), alert_type(char), message(text), created_at(ts), acknowledged(bool), workspace_id(text), acknowledged_workspace(bool), resource(text) diff --git a/backend/tests/ai_shared_artifacts.rs b/backend/tests/ai_shared_artifacts.rs new file mode 100644 index 0000000000..17dd313755 --- /dev/null +++ b/backend/tests/ai_shared_artifacts.rs @@ -0,0 +1,184 @@ +//! Shared AI session artifacts: one link per author and artifact, readable by any workspace +//! member until its retention window passes, and removable only by its author or an admin. +//! +//! Expiry is enforced on read as well as by the monitor's sweep, so a share past its window must +//! not be served in the gap before the sweep reaches it. + +use serde_json::{json, Value}; +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +const ADMIN: &str = "Bearer SECRET_TOKEN"; +const MEMBER: &str = "Bearer SECRET_TOKEN_2"; + +async fn share( + client: &reqwest::Client, + base: &str, + token: &str, + content: &str, +) -> anyhow::Result { + let resp = client + .post(format!("{base}/share")) + .header("Authorization", token) + .json(&json!({ + "artifact_id": "plan:session-1", + "name": "Plan", + "kind": "md", + "version": 1, + "content": content, + })) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + Ok(resp.json().await?) +} + +#[sqlx::test(fixtures("base", "ai_shared_artifacts"))] +async fn shared_artifact_is_served_to_members_until_it_expires( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let base = format!( + "http://localhost:{}/api/w/test-workspace/ai/shared_artifacts", + server.addr.port() + ); + let client = reqwest::Client::new(); + + let first = share(&client, &base, MEMBER, "draft").await?; + let second = share(&client, &base, MEMBER, "final").await?; + assert_eq!(first["id"], second["id"], "re-sharing minted a second link"); + let id = second["id"].as_str().unwrap(); + + let resp = client + .get(format!("{base}/get/{id}")) + .header("Authorization", ADMIN) + .send() + .await?; + assert_eq!(resp.status(), 200); + let body: Value = resp.json().await?; + assert_eq!(body["content"], "final"); + + // The handlers read through the raw pool, so the workspace in the URL is the only thing + // scoping a share: a member of another workspace must not reach it by id through theirs. + let resp = client + .get(format!( + "http://localhost:{}/api/w/test-workspace-2/ai/shared_artifacts/get/{id}", + server.addr.port() + )) + .header("Authorization", ADMIN) + .send() + .await?; + assert_eq!( + resp.status(), + 404, + "a share was served through another workspace's path" + ); + + sqlx::query( + "UPDATE ai_shared_artifact SET shared_at = now() - ($1::bigint + 60) * interval '1 second'", + ) + .bind(windmill_common::ai_shared_artifact_retention_secs()) + .execute(&db) + .await?; + + let resp = client + .get(format!("{base}/get/{id}")) + .header("Authorization", ADMIN) + .send() + .await?; + assert_eq!(resp.status(), 404, "an expired share was served"); + + let status: Value = client + .get(format!("{base}/status?artifact_id=plan:session-1")) + .header("Authorization", MEMBER) + .send() + .await? + .json() + .await?; + assert!( + status.get("share").is_none(), + "an expired share was reported live: {status}" + ); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn only_the_author_or_an_admin_can_unshare(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let base = format!( + "http://localhost:{}/api/w/test-workspace/ai/shared_artifacts", + server.addr.port() + ); + let client = reqwest::Client::new(); + + let shared = share(&client, &base, ADMIN, "admin's plan").await?; + let id = shared["id"].as_str().unwrap(); + + let resp = client + .delete(format!("{base}/delete/{id}")) + .header("Authorization", MEMBER) + .send() + .await?; + assert_eq!(resp.status(), 404); + let remaining: i64 = sqlx::query_scalar("SELECT count(*) FROM ai_shared_artifact") + .fetch_one(&db) + .await?; + assert_eq!(remaining, 1, "a member deleted someone else's share"); + + let member_share = share(&client, &base, MEMBER, "member's plan").await?; + let resp = client + .delete(format!( + "{base}/delete/{}", + member_share["id"].as_str().unwrap() + )) + .header("Authorization", ADMIN) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + + Ok(()) +} + +/// The id is compared against a `VARCHAR(255)` column on every route that takes one, and a +/// NUL in it would otherwise reach Postgres and come back as a 500. +#[sqlx::test(fixtures("base"))] +async fn a_malformed_artifact_id_is_refused_on_every_route( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let base = format!( + "http://localhost:{}/api/w/test-workspace/ai/shared_artifacts", + server.addr.port() + ); + let client = reqwest::Client::new(); + + for bad_id in ["", "a\0b", &"x".repeat(256)] { + let resp = client + .get(format!("{base}/status")) + .query(&[("artifact_id", bad_id)]) + .header("Authorization", MEMBER) + .send() + .await?; + assert_eq!(resp.status(), 400, "status accepted {bad_id:?}"); + + let resp = client + .post(format!("{base}/share")) + .header("Authorization", MEMBER) + .json(&json!({ + "artifact_id": bad_id, + "name": "Plan", + "kind": "md", + "version": 1, + "content": "x", + })) + .send() + .await?; + assert_eq!(resp.status(), 400, "share accepted {bad_id:?}"); + } + + Ok(()) +} diff --git a/backend/tests/fixtures/ai_shared_artifacts.sql b/backend/tests/fixtures/ai_shared_artifacts.sql new file mode 100644 index 0000000000..5f745ed0cf --- /dev/null +++ b/backend/tests/fixtures/ai_shared_artifacts.sql @@ -0,0 +1,17 @@ +-- Layers on `base`: a second workspace the superadmin `test-user` is also a member of, so a +-- share can be requested through the wrong workspace's path by a caller the route accepts. + +INSERT INTO workspace (id, name, owner) VALUES + ('test-workspace-2', 'test-workspace-2', 'test-user'); + +INSERT INTO workspace_key(workspace_id, kind, key) VALUES + ('test-workspace-2', 'cloud', 'test-key-2'); + +INSERT INTO workspace_settings (workspace_id) VALUES + ('test-workspace-2'); + +INSERT INTO group_ (workspace_id, name, summary, extra_perms) VALUES + ('test-workspace-2', 'all', 'All users', '{}'); + +INSERT INTO usr(workspace_id, email, username, is_admin, role) VALUES + ('test-workspace-2', 'test@windmill.dev', 'test-user', true, 'Admin'); diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index aee198e92a..411d5f658c 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -13191,6 +13191,134 @@ paths: type: boolean description: more buckets matched than were returned, so summing them under-reports + /w/{workspace}/ai/shared_artifacts/share: + post: + summary: share an AI session artifact with the workspace + description: > + Stores a read-only copy that any member of the workspace can open by id. Sharing the + same artifact again updates that copy, keeps its id, and restarts its retention window. + operationId: shareAiArtifact + tags: + - ai + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - artifact_id + - name + - kind + - version + - content + properties: + artifact_id: + type: string + description: the artifact's id in the author's session + name: + type: string + kind: + type: string + enum: [md, html] + version: + type: integer + minimum: 1 + content: + type: string + responses: + "200": + description: the shared copy + content: + application/json: + schema: + $ref: "#/components/schemas/SharedAiArtifactInfo" + + /w/{workspace}/ai/shared_artifacts/status: + get: + summary: get the calling user's share of one of their AI session artifacts + operationId: getAiArtifactShareStatus + tags: + - ai + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: artifact_id + in: query + required: true + schema: + type: string + responses: + "200": + description: the live share, if any, and how long shares last + content: + application/json: + schema: + type: object + required: + - retention_secs + properties: + retention_secs: + type: integer + share: + $ref: "#/components/schemas/SharedAiArtifactInfo" + + /w/{workspace}/ai/shared_artifacts/get/{id}: + get: + summary: get a shared AI session artifact + operationId: getSharedAiArtifact + tags: + - ai + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: id + in: path + required: true + schema: + type: string + format: uuid + responses: + "200": + description: the shared artifact + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/SharedAiArtifactInfo" + - type: object + required: + - content + - can_unshare + properties: + content: + type: string + can_unshare: + type: boolean + description: whether the caller authored the share or is a workspace admin + + /w/{workspace}/ai/shared_artifacts/delete/{id}: + delete: + summary: stop sharing an AI session artifact + operationId: unshareAiArtifact + tags: + - ai + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: id + in: path + required: true + schema: + type: string + format: uuid + responses: + "200": + description: share deleted + content: + text/plain: + schema: + type: string + /w/{workspace}/apps/get_data/v/{secretWithExtension}: get: summary: get raw app data by @@ -28256,6 +28384,36 @@ components: - output_tokens - requests + SharedAiArtifactInfo: + type: object + properties: + id: + type: string + format: uuid + name: + type: string + kind: + type: string + enum: [md, html] + version: + type: integer + created_by: + type: string + shared_at: + type: string + format: date-time + expires_at: + type: string + format: date-time + required: + - id + - name + - kind + - version + - created_by + - shared_at + - expires_at + InstanceAIProviderSummary: type: object properties: diff --git a/backend/windmill-api/src/ai.rs b/backend/windmill-api/src/ai.rs index e70cc19544..8101064c3d 100644 --- a/backend/windmill-api/src/ai.rs +++ b/backend/windmill-api/src/ai.rs @@ -518,6 +518,10 @@ pub fn workspaced_service() -> Router { // could make the server allocate and parse an arbitrarily large one. // Sized well above a full batch of the shape below. .layer(DefaultBodyLimit::max(AI_USAGE_BODY_LIMIT)), + ) + .nest( + "/shared_artifacts", + crate::ai_shared_artifacts::workspaced_service(), ); #[cfg(feature = "bedrock")] diff --git a/backend/windmill-api/src/ai_shared_artifacts.rs b/backend/windmill-api/src/ai_shared_artifacts.rs new file mode 100644 index 0000000000..36ad0ecd2e --- /dev/null +++ b/backend/windmill-api/src/ai_shared_artifacts.rs @@ -0,0 +1,322 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2026 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Read-only copies of AI session artifacts, shared with the workspace by their author. +//! +//! Artifacts live in the author's browser; a row here exists only because the author asked +//! for a link. Every read filters on the retention window as well as the monitor sweeping +//! it, so a share past its window is never served in the gap before the sweep reaches it. + +use crate::db::{ApiAuthed, DB}; +use axum::{ + extract::{DefaultBodyLimit, Extension, Json, Path, Query}, + routing::{delete, get, post}, + Router, +}; +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; +use windmill_audit::{audit_oss::audit_log, ActionKind}; +use windmill_common::{ + ai_shared_artifact_retention_secs, + error::{Error, JsonResult, Result}, +}; + +/// The frontend's `MAX_ARTIFACT_BYTES`: no artifact the viewer holds is larger. +const MAX_CONTENT_BYTES: usize = 256 * 1024; +const MAX_NAME_CHARS: usize = 255; +const MAX_ARTIFACT_ID_CHARS: usize = 255; +/// JSON escapes a control character into six bytes, so a full-size artifact made entirely of +/// them still fits. +const SHARE_BODY_LIMIT: usize = MAX_CONTENT_BYTES * 6 + 64 * 1024; + +pub fn workspaced_service() -> Router { + Router::new() + .route( + "/share", + post(share_artifact).layer(DefaultBodyLimit::max(SHARE_BODY_LIMIT)), + ) + .route("/status", get(get_share_status)) + .route("/get/{id}", get(get_shared_artifact)) + .route("/delete/{id}", delete(unshare_artifact)) +} + +#[derive(Deserialize, Clone, Copy)] +#[serde(rename_all = "lowercase")] +enum ArtifactKind { + Md, + Html, +} + +impl ArtifactKind { + fn as_str(self) -> &'static str { + match self { + ArtifactKind::Md => "md", + ArtifactKind::Html => "html", + } + } +} + +#[derive(Deserialize)] +struct ShareArtifact { + artifact_id: String, + name: String, + kind: ArtifactKind, + version: i32, + content: String, +} + +#[derive(Serialize)] +struct SharedArtifactInfo { + id: Uuid, + name: String, + kind: String, + version: i32, + created_by: String, + shared_at: DateTime, + expires_at: DateTime, +} + +#[derive(Serialize)] +struct SharedArtifact { + #[serde(flatten)] + info: SharedArtifactInfo, + content: String, + /// Whether the caller may stop sharing it: its author, or a workspace admin. + can_unshare: bool, +} + +#[derive(Serialize)] +struct ShareStatus { + retention_secs: i64, + #[serde(skip_serializing_if = "Option::is_none")] + share: Option, +} + +#[derive(Deserialize)] +struct ShareStatusQuery { + artifact_id: String, +} + +fn expires_at(shared_at: DateTime, retention_secs: i64) -> DateTime { + shared_at + chrono::Duration::seconds(retention_secs) +} + +/// The browser-side artifact id, as every handler that takes one must check it: it is compared +/// against a `VARCHAR(255)` column, and Postgres answers a NUL in a text parameter with an +/// opaque 500. +fn check_artifact_id(artifact_id: &str) -> Result<()> { + if artifact_id.is_empty() || artifact_id.chars().count() > MAX_ARTIFACT_ID_CHARS { + return Err(Error::BadRequest(format!( + "Artifact id must be between 1 and {MAX_ARTIFACT_ID_CHARS} characters" + ))); + } + if artifact_id.contains('\0') { + return Err(Error::BadRequest( + "Artifact id cannot contain NUL characters".to_string(), + )); + } + Ok(()) +} + +/// Share an artifact, or move the caller's existing link for it to this content. Re-sharing +/// keeps the link and restarts its retention window. +async fn share_artifact( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Json(payload): Json, +) -> JsonResult { + let name = payload.name.trim(); + if name.is_empty() || name.chars().count() > MAX_NAME_CHARS { + return Err(Error::BadRequest(format!( + "Artifact name must be between 1 and {MAX_NAME_CHARS} characters" + ))); + } + check_artifact_id(&payload.artifact_id)?; + if payload.content.len() > MAX_CONTENT_BYTES { + return Err(Error::BadRequest(format!( + "Artifact content is {} bytes, above the {MAX_CONTENT_BYTES} byte limit", + payload.content.len() + ))); + } + if payload.version < 1 { + return Err(Error::BadRequest( + "Artifact version must be at least 1".to_string(), + )); + } + // Postgres rejects NUL in text columns with an opaque 500. + if name.contains('\0') || payload.content.contains('\0') { + return Err(Error::BadRequest( + "Artifact name and content cannot contain NUL characters".to_string(), + )); + } + + let mut tx = db.begin().await?; + let row = sqlx::query!( + r#"INSERT INTO ai_shared_artifact + (workspace_id, artifact_id, email, created_by, name, kind, version, content) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) + ON CONFLICT (workspace_id, email, artifact_id) DO UPDATE + SET created_by = EXCLUDED.created_by, + name = EXCLUDED.name, + kind = EXCLUDED.kind, + version = EXCLUDED.version, + content = EXCLUDED.content, + shared_at = now() + RETURNING id, shared_at, (xmax = 0) AS "inserted!""#, + &w_id, + &payload.artifact_id, + &authed.email, + &authed.username, + name, + payload.kind.as_str(), + payload.version, + &payload.content, + ) + .fetch_one(&mut *tx) + .await?; + + let id = row.id.to_string(); + audit_log( + &mut *tx, + &authed, + "ai.shared_artifacts.share", + if row.inserted { + ActionKind::Create + } else { + ActionKind::Update + }, + &w_id, + Some(&id), + Some([("name", name)].into()), + ) + .await?; + tx.commit().await?; + + Ok(Json(SharedArtifactInfo { + id: row.id, + name: name.to_string(), + kind: payload.kind.as_str().to_string(), + version: payload.version, + created_by: authed.username.clone(), + shared_at: row.shared_at, + expires_at: expires_at(row.shared_at, ai_shared_artifact_retention_secs()), + })) +} + +/// The caller's own live share of one of their artifacts, if any, and how long a share lasts. +async fn get_share_status( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Query(query): Query, +) -> JsonResult { + check_artifact_id(&query.artifact_id)?; + let retention_secs = ai_shared_artifact_retention_secs(); + let share = sqlx::query!( + "SELECT id, name, kind, version, created_by, shared_at FROM ai_shared_artifact + WHERE workspace_id = $1 AND email = $2 AND artifact_id = $3 + AND shared_at > now() - ($4::bigint::text || ' s')::interval", + &w_id, + &authed.email, + &query.artifact_id, + retention_secs, + ) + .fetch_optional(&db) + .await? + .map(|r| SharedArtifactInfo { + id: r.id, + name: r.name, + kind: r.kind, + version: r.version, + created_by: r.created_by, + shared_at: r.shared_at, + expires_at: expires_at(r.shared_at, retention_secs), + }); + + Ok(Json(ShareStatus { retention_secs, share })) +} + +/// Any member of the workspace may read a live share: that is what sharing it granted. +async fn get_shared_artifact( + authed: ApiAuthed, + Extension(db): Extension, + Path((w_id, id)): Path<(String, Uuid)>, +) -> JsonResult { + let retention_secs = ai_shared_artifact_retention_secs(); + let row = sqlx::query!( + "SELECT id, email, name, kind, version, created_by, content, shared_at + FROM ai_shared_artifact + WHERE workspace_id = $1 AND id = $2 + AND shared_at > now() - ($3::bigint::text || ' s')::interval", + &w_id, + id, + retention_secs, + ) + .fetch_optional(&db) + .await? + .ok_or_else(|| { + Error::NotFound(format!( + "Shared artifact {id} not found: it may have expired or been unshared" + )) + })?; + + Ok(Json(SharedArtifact { + can_unshare: row.email == authed.email || authed.is_admin, + content: row.content, + info: SharedArtifactInfo { + id: row.id, + name: row.name, + kind: row.kind, + version: row.version, + created_by: row.created_by, + shared_at: row.shared_at, + expires_at: expires_at(row.shared_at, retention_secs), + }, + })) +} + +async fn unshare_artifact( + authed: ApiAuthed, + Extension(db): Extension, + Path((w_id, id)): Path<(String, Uuid)>, +) -> Result { + let mut tx = db.begin().await?; + let name = sqlx::query_scalar!( + "DELETE FROM ai_shared_artifact + WHERE workspace_id = $1 AND id = $2 AND (email = $3 OR $4::bool) + RETURNING name", + &w_id, + id, + &authed.email, + authed.is_admin, + ) + .fetch_optional(&mut *tx) + .await? + .ok_or_else(|| { + Error::NotFound(format!( + "Shared artifact {id} not found, or not shared by you" + )) + })?; + + let id_str = id.to_string(); + audit_log( + &mut *tx, + &authed, + "ai.shared_artifacts.unshare", + ActionKind::Delete, + &w_id, + Some(&id_str), + Some([("name", name.as_str())].into()), + ) + .await?; + tx.commit().await?; + + Ok(format!("Stopped sharing {name}")) +} diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index bd6fda015b..fc90852ae6 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -69,6 +69,7 @@ mod ai; #[cfg(feature = "private")] mod ai_free_tier_ee; mod ai_free_tier_oss; +mod ai_shared_artifacts; mod apps; mod apps_raw_bundle; pub use apps::invalidate_app_policy_cache; diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index c0ed63cd53..49efa26f60 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -577,6 +577,7 @@ pub const ENV_SETTINGS: &[&str] = &[ "OTEL_RESOURCE_ATTRIBUTES", "OTEL_JOB_LOGS", "OTEL_TRACES_RETENTION_SECS", + "AI_SHARED_ARTIFACT_RETENTION_SECS", "DISABLE_S3_STORE", "PG_SCHEMA", "PG_LISTENER_REFRESH_PERIOD_SECS", diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index dce40048f5..8e17ec676e 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -151,6 +151,7 @@ pub const DEFAULT_HUB_BASE_URL: &str = "https://hub.windmill.dev"; pub const PRIVATE_HUB_MIN_VERSION: i32 = 10_000_000; pub const DEFAULT_SERVICE_LOG_RETENTION_SECS: i64 = 60 * 60 * 24 * 14; // 2 weeks retention period for logs pub const DEFAULT_OTEL_TRACES_RETENTION_SECS: i64 = 60 * 60 * 24 * 7; // 1 week retention period for HTTP request spans +pub const DEFAULT_AI_SHARED_ARTIFACT_RETENTION_SECS: i64 = 60 * 60 * 24 * 30; pub const WM_DEPLOYERS_GROUP: &str = "wm_deployers"; /// A century. Every consumer has to survive `now - retention`, and the ceilings are much lower @@ -229,6 +230,13 @@ pub fn service_log_retention_secs() -> i64 { SERVICE_LOG_RETENTION_SECS.load(std::sync::atomic::Ordering::Relaxed) } +/// How long a shared AI session artifact stays viewable, in seconds, counted from the last time +/// its author shared it. Read by both the API, which stops serving an expired share, and the +/// monitor, which deletes it — so both must agree, which is why they share this one reader. +pub fn ai_shared_artifact_retention_secs() -> i64 { + *AI_SHARED_ARTIFACT_RETENTION_SECS +} + /// Canonical form of a base URL, used as one of the inputs to the offline-license /// instance hash (`compute_instance_hash`). /// @@ -476,6 +484,15 @@ lazy_static::lazy_static! { /// [`set_otel_traces_retention_secs`] is the only writer, [`otel_traces_retention_secs`] the /// only reader. static ref OTEL_TRACES_RETENTION_SECS: AtomicI64 = AtomicI64::new(DEFAULT_OTEL_TRACES_RETENTION_SECS); + /// Read it with [`ai_shared_artifact_retention_secs`]. + static ref AI_SHARED_ARTIFACT_RETENTION_SECS: i64 = clamp_retention_secs( + std::env::var("AI_SHARED_ARTIFACT_RETENTION_SECS") + .ok() + .and_then(|s| s.parse::().ok()) + .unwrap_or(DEFAULT_AI_SHARED_ARTIFACT_RETENTION_SECS), + DEFAULT_AI_SHARED_ARTIFACT_RETENTION_SECS, + "AI shared artifact", + ); pub static ref MONITOR_LOGS_ON_OBJECT_STORE: AtomicBool = AtomicBool::new(false); diff --git a/frontend/src/lib/components/copilot/chat/LinkRenderer.svelte b/frontend/src/lib/components/copilot/chat/LinkRenderer.svelte index e93017b095..364e7fceaf 100644 --- a/frontend/src/lib/components/copilot/chat/LinkRenderer.svelte +++ b/frontend/src/lib/components/copilot/chat/LinkRenderer.svelte @@ -13,6 +13,7 @@ type WindmillItemKind, type WorkspaceItemTargetKind } from './workspaceItems.svelte' + import { safeHref } from './safeHref' type Props = { href?: string @@ -45,6 +46,8 @@ const previewAction = $derived(available?.type === 'open_item_preview' ? available : undefined) const drawerAction = $derived(available?.type === 'open_created_resource' ? available : undefined) + const allowedHref = $derived(safeHref(href, window.location.href)) + const modifier = newTabModifier() const hint = $derived( @@ -68,7 +71,7 @@ } -{#if href} +{#if allowedHref} {#if wmKind} {:else} - + {@render children?.()} {/if} +{:else} + + {@render children?.()} {/if} diff --git a/frontend/src/lib/components/copilot/chat/artifacts/ArtifactBody.svelte b/frontend/src/lib/components/copilot/chat/artifacts/ArtifactBody.svelte new file mode 100644 index 0000000000..7db5604775 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/artifacts/ArtifactBody.svelte @@ -0,0 +1,46 @@ + + + +{#if source} + + {#key content} + + {/key} +{:else} + +
    +
    + +
    +{/if} diff --git a/frontend/src/lib/components/copilot/chat/artifacts/ArtifactExportButton.svelte b/frontend/src/lib/components/copilot/chat/artifacts/ArtifactExportButton.svelte new file mode 100644 index 0000000000..9fc5f767b4 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/artifacts/ArtifactExportButton.svelte @@ -0,0 +1,42 @@ + + + + diff --git a/frontend/src/lib/components/copilot/chat/artifacts/ArtifactShareButton.svelte b/frontend/src/lib/components/copilot/chat/artifacts/ArtifactShareButton.svelte new file mode 100644 index 0000000000..4924473e98 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/artifacts/ArtifactShareButton.svelte @@ -0,0 +1,196 @@ + + + + {#snippet trigger()} + + {/snippet} + {#snippet content()} +
    +
    + Share with workspace + + {#if share} + Members of {workspace} can open a read-only copy of v{share.version} with this link. + {:else} + Members of {workspace} will be able to open a read-only copy of v{version} with a link. + {/if} + {#if status.current} + The copy is deleted {formatRetention(status.current.retention_secs)} after it is shared. + {/if} + +
    + + {#if share && url} +
    + + + Expires {displayDate(share.expires_at)} + +
    + {#if change} +
    + + {#if change === 'newer'} + The link shows v{share.version}; v{version} is on screen. + {:else if change === 'older'} + The link shows v{share.version}, newer than the v{version} on screen. + {:else} + The link still shows the old name, “{share.name}”. + {/if} + + +
    + {/if} +
    + +
    + {:else} +
    + +
    + {/if} +
    + {/snippet} +     diff --git a/frontend/src/lib/components/copilot/chat/artifacts/ArtifactViewer.svelte b/frontend/src/lib/components/copilot/chat/artifacts/ArtifactViewer.svelte index ebc46178d0..8331825ee2 100644 --- a/frontend/src/lib/components/copilot/chat/artifacts/ArtifactViewer.svelte +++ b/frontend/src/lib/components/copilot/chat/artifacts/ArtifactViewer.svelte @@ -1,23 +1,14 @@
    @@ -169,27 +144,22 @@ {/if}
    - - + /> + {#if canPreview} {#if restoringPin} - {:else if source} - - {#key `${artifact.id}:${pinnedContent ? `v${pinnedContent.version}` : artifact.updatedAt}`} - - {/key} {:else} - -
    -
    - -
    + {/if}
    diff --git a/frontend/src/lib/components/copilot/chat/artifacts/artifactSharing.test.ts b/frontend/src/lib/components/copilot/chat/artifacts/artifactSharing.test.ts new file mode 100644 index 0000000000..115343117a --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/artifacts/artifactSharing.test.ts @@ -0,0 +1,21 @@ +import { describe, expect, it, vi } from 'vitest' + +vi.mock('$lib/base', () => ({ base: '' })) + +import { shareWorkspaceId } from './artifactSharing' + +describe('shareWorkspaceId', () => { + it('shares a fork session into the topmost workspace the user still belongs to', () => { + const workspaces = [ + { id: 'prod' }, + { id: 'wm-fork-a', parent_workspace_id: 'prod' }, + { id: 'wm-fork-b', parent_workspace_id: 'wm-fork-a' } + ] + expect(shareWorkspaceId('wm-fork-b', workspaces)).toBe('prod') + }) + + it('stops below a parent the user is not a member of', () => { + const workspaces = [{ id: 'wm-fork-a', parent_workspace_id: 'prod' }] + expect(shareWorkspaceId('wm-fork-a', workspaces)).toBe('wm-fork-a') + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/artifacts/artifactSharing.ts b/frontend/src/lib/components/copilot/chat/artifacts/artifactSharing.ts new file mode 100644 index 0000000000..57b08cba9b --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/artifacts/artifactSharing.ts @@ -0,0 +1,42 @@ +import { base } from '$lib/base' + +/** + * The workspace a share from `workspaceId` lands in: the topmost ancestor the user still + * belongs to. A session often runs in a fork, whose members are its creator alone, so a link + * minted there would reach nobody — and it would be deleted with the fork. + */ +export function shareWorkspaceId( + workspaceId: string, + workspaces: { id: string; parent_workspace_id?: string | null }[] +): string { + let current = workspaceId + const seen = new Set([current]) + for (;;) { + const parent = workspaces.find((w) => w.id === current)?.parent_workspace_id + if (!parent || seen.has(parent) || !workspaces.some((w) => w.id === parent)) return current + seen.add(parent) + current = parent + } +} + +export function sharedArtifactUrl(workspaceId: string, shareId: string): string { + return `${window.location.origin}${base}/shared_artifacts/${encodeURIComponent( + shareId + )}?workspace=${encodeURIComponent(workspaceId)}` +} + +/** "30 days", "12 hours": the retention window in the largest whole unit it fills. */ +export function formatRetention(secs: number): string { + const units: [string, number][] = [ + ['day', 86400], + ['hour', 3600], + ['minute', 60] + ] + for (const [unit, size] of units) { + if (secs >= size) { + const n = Math.floor(secs / size) + return `${n} ${unit}${n === 1 ? '' : 's'}` + } + } + return `${secs} second${secs === 1 ? '' : 's'}` +} diff --git a/frontend/src/lib/components/copilot/chat/safeHref.test.ts b/frontend/src/lib/components/copilot/chat/safeHref.test.ts new file mode 100644 index 0000000000..5204ebe994 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/safeHref.test.ts @@ -0,0 +1,33 @@ +import { describe, expect, it } from 'vitest' +import { safeHref } from './safeHref' + +const BASE = 'https://app.example.com/sessions?workspace=demo' + +describe('safeHref', () => { + it.each([ + 'https://windmill.dev/docs', + 'http://localhost:3000/', + 'mailto:someone@example.com', + '/runs/abc', + '#anchor', + 'docs/page' + ])('keeps %s', (href) => { + expect(safeHref(href, BASE)).toBe(href) + }) + + it.each([ + 'javascript:alert(1)', + 'JavaScript:alert(1)', + ' javascript:alert(1)', + 'data:text/html,', + 'vbscript:msgbox', + 'file:///etc/passwd' + ])('drops %s', (href) => { + expect(safeHref(href, BASE)).toBeUndefined() + }) + + it('drops a missing or empty href', () => { + expect(safeHref(undefined, BASE)).toBeUndefined() + expect(safeHref('', BASE)).toBeUndefined() + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/safeHref.ts b/frontend/src/lib/components/copilot/chat/safeHref.ts new file mode 100644 index 0000000000..3acdabb464 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/safeHref.ts @@ -0,0 +1,17 @@ +const SAFE_PROTOCOLS = ['http:', 'https:', 'mailto:'] + +/** + * The href a rendered markdown link may carry, or undefined for one that must be dropped. + * + * Markdown reaches the chat renderers from a model, and from another member for a shared + * artifact, and `svelte-exmarkdown` passes `javascript:` and `data:` hrefs through untouched. + * Relative links resolve against `base` (the page), so they stay. + */ +export function safeHref(href: string | undefined, base: string): string | undefined { + if (!href) return undefined + try { + return SAFE_PROTOCOLS.includes(new URL(href, base).protocol) ? href : undefined + } catch { + return undefined + } +} diff --git a/frontend/src/routes/(root)/(logged)/shared_artifacts/[id]/+page.svelte b/frontend/src/routes/(root)/(logged)/shared_artifacts/[id]/+page.svelte new file mode 100644 index 0000000000..80449e6244 --- /dev/null +++ b/frontend/src/routes/(root)/(logged)/shared_artifacts/[id]/+page.svelte @@ -0,0 +1,144 @@ + + +
    +
    + {#if artifact} +
    +
    +
    + +

    + {artifact.name} +

    +
    + + Shared by {artifact.created_by} · v{artifact.version} · {displayDate( + artifact.shared_at + )} · expires {displayDate(artifact.expires_at)} + +
    +
    + {#if artifact.can_unshare} + + {/if} + + {#if artifact.kind === 'md'} + (showSource = v === 'source')} + > + {#snippet children({ item })} + + + {/snippet} + + {/if} +
    +
    +
    + +
    + {:else if shared.current?.state === 'gone'} + + {:else if shared.current?.state === 'error'} + + {:else} + + {/if} +
    +
    diff --git a/frontend/src/routes/(root)/(logged)/shared_artifacts/[id]/+page.ts b/frontend/src/routes/(root)/(logged)/shared_artifacts/[id]/+page.ts new file mode 100644 index 0000000000..efbac8862d --- /dev/null +++ b/frontend/src/routes/(root)/(logged)/shared_artifacts/[id]/+page.ts @@ -0,0 +1,5 @@ +export function load() { + return { + stuff: { title: 'Shared artifact' } + } +} From 5dcf40cb4f9d1d4d738c81cf57c3056e08eeadf0 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 14 Sep 2026 23:06:36 +0200 Subject: [PATCH 117/144] chore: move the EE pin forward to the commit that claims pending oauth accounts (#11127) Claude-Session: https://claude.ai/code/session_01GGciSSE5EFMiDf1dFWQq5M Co-authored-by: Claude Fable 5.1 --- backend/ee-repo-ref.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index a02a129b97..062211f925 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -04a9f1efb4a52c79fcd20258b34780c86103d27f +1ba6fe83451f0a1f8fafe04b7187087d51e0f769 From 75ee497011dca076de0923ded9da8e23e08bfb84 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 14 Sep 2026 23:16:37 +0200 Subject: [PATCH 118/144] fix(cli): stage a rewritten shared lockfile on git-sync deploy push (#11126) * fix(cli): stage a rewritten shared lockfile on git-sync deploy push Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01GGciSSE5EFMiDf1dFWQq5M * test(cli): pin that a swept shared lockfile is committed as a deletion Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01GGciSSE5EFMiDf1dFWQq5M * chore: bump the git sync hub script to 28969 Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01GGciSSE5EFMiDf1dFWQq5M --------- Co-authored-by: Claude Fable 5.1 --- backend/windmill-common/src/workspaces.rs | 2 +- cli/src/utils/git.ts | 6 ++ cli/test/gitsync_deploy_push_unit.test.ts | 98 +++++++++++++++++++++++ 3 files changed, 105 insertions(+), 1 deletion(-) create mode 100644 cli/test/gitsync_deploy_push_unit.test.ts diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index ad040c5905..3b00a87920 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -183,7 +183,7 @@ pub enum ObjectType { DatatableMigration, } -pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28958/sync-script-to-git-repo-windmill"; +pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28969/sync-script-to-git-repo-windmill"; /// Hub script that applies a repository's state back into a workspace /// (the repo → Windmill / "pull" direction). Same script the UI runs from diff --git a/cli/src/utils/git.ts b/cli/src/utils/git.ts index 5ca06f427a..354e15c36e 100644 --- a/cli/src/utils/git.ts +++ b/cli/src/utils/git.ts @@ -1,6 +1,7 @@ import * as log from "../core/log.ts"; import { execSync, spawnSync } from "node:child_process"; import { WM_FORK_PREFIX } from "../core/constants.ts"; +import { SHARED_LOCK_DIR } from "./script_common.ts"; // Fork *workspace id* prefix ("wm-fork-"). WM_FORK_PREFIX is the *branch* // prefix ("wm-fork") used inside the wm-fork// branch name. @@ -584,6 +585,11 @@ export function gitSyncDeployPush(params: { git(["add", "wmill-lock.yaml", `${parent_path}**`], { allowFail: true }); } } + // A shared lockfile (`dedupeLockfiles`) lives under `locks/`, outside every + // item's path glob, and the pull rewrites it when a deployed script's lock + // changed. `-A` also stages the deletion of a swept one; the add fails only + // when nothing under `locks/` exists or is tracked. + git(["add", "-A", "--", SHARED_LOCK_DIR], { allowFail: true }); // `git diff --cached --quiet` exits 1 iff there is something staged. const staged = git(["diff", "--cached", "--quiet"], { allowFail: true }); diff --git a/cli/test/gitsync_deploy_push_unit.test.ts b/cli/test/gitsync_deploy_push_unit.test.ts new file mode 100644 index 0000000000..f87e86bcc3 --- /dev/null +++ b/cli/test/gitsync_deploy_push_unit.test.ts @@ -0,0 +1,98 @@ +import { expect, test } from "bun:test"; +import { execFileSync } from "node:child_process"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { gitSyncDeployPush } from "../src/utils/git.ts"; + +function git(cwd: string, ...args: string[]): string { + return execFileSync("git", args, { cwd, encoding: "utf8" }).trim(); +} + +// A seeded clone of a bare remote, with `files` committed on main. +async function seededClone( + files: Record, +): Promise<{ bare: string; work: string }> { + const bare = await mkdtemp(join(tmpdir(), "wmill_deploy_push_bare_")); + execFileSync("git", ["init", "--quiet", "--bare", "--initial-branch=main", bare]); + const work = await mkdtemp(join(tmpdir(), "wmill_deploy_push_work_")); + git(work, "init", "--quiet", "--initial-branch=main"); + git(work, "config", "user.email", "seed@windmill.dev"); + git(work, "config", "user.name", "seed"); + for (const [path, content] of Object.entries(files)) { + await mkdir(join(work, path, ".."), { recursive: true }); + await writeFile(join(work, path), content); + } + git(work, "add", "-A"); + git(work, "commit", "--quiet", "-m", "seed"); + git(work, "remote", "add", "origin", `file://${bare}`); + git(work, "push", "--quiet", "-u", "origin", "main"); + return { bare, work }; +} + +function deployPushIn(work: string, path: string) { + const cwd = process.cwd(); + process.chdir(work); + try { + return gitSyncDeployPush({ + items: [{ path_type: "script", path, commit_msg: `deploy ${path}` }], + authorName: "windmill", + authorEmail: "windmill@windmill.dev", + }); + } finally { + process.chdir(cwd); + } +} + +test("a rewritten shared lockfile is committed with the deployed item", async () => { + const { bare, work } = await seededClone({ + "wmill-lock.yaml": "locks: {}\n", + "f/dd/a.script.yaml": "lock: '!inline locks/requirements.in.lock'\n", + "locks/requirements.in.lock": "requests==2.31.0\n", + }); + // What the deploy callback's pull leaves behind after `f/dd/a` was relocked: + // the item's own files are unchanged, only the shared file moved. + await writeFile(join(work, "locks/requirements.in.lock"), "requests==2.32.3\n"); + + expect(deployPushIn(work, "f/dd/a").pushed).toBe(true); + expect(git(work, "show", "--name-only", "--format=", "HEAD")).toBe( + "locks/requirements.in.lock", + ); + expect(git(bare, "cat-file", "-p", "main:locks/requirements.in.lock")).toBe( + "requests==2.32.3", + ); + + await rm(bare, { recursive: true, force: true }); + await rm(work, { recursive: true, force: true }); +}); + +test("a swept shared lockfile is committed as a deletion", async () => { + const { bare, work } = await seededClone({ + "wmill-lock.yaml": "locks: {}\n", + "f/dd/a.script.yaml": "lock: '!inline f/dd/a.script.lock'\n", + "f/dd/a.script.lock": "requests==2.31.0\n", + "locks/requirements.in.lock": "requests==2.31.0\n", + }); + // The pull removed the last shared lockfile, and `locks/` with it. + await rm(join(work, "locks"), { recursive: true, force: true }); + + expect(deployPushIn(work, "f/dd/a").pushed).toBe(true); + expect(git(bare, "ls-tree", "--name-only", "main", "locks/")).toBe(""); + + await rm(bare, { recursive: true, force: true }); + await rm(work, { recursive: true, force: true }); +}); + +test("a repository without shared lockfiles is left alone", async () => { + const { bare, work } = await seededClone({ + "wmill-lock.yaml": "locks: {}\n", + "f/dd/a.script.yaml": "lock: '!inline f/dd/a.script.lock'\n", + "f/dd/a.script.lock": "requests==2.31.0\n", + }); + + expect(deployPushIn(work, "f/dd/a").pushed).toBe(false); + expect(git(bare, "rev-parse", "main")).toBe(git(work, "rev-parse", "HEAD")); + + await rm(bare, { recursive: true, force: true }); + await rm(work, { recursive: true, force: true }); +}); From e3e638f7f587f0ee090d06436575b65a0860a855 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 15 Sep 2026 09:45:10 +0200 Subject: [PATCH 119/144] fix: skip instance group members that are not email addresses (#11128) * fix: skip instance group members that are not email addresses * fix: keep provisioned members whose address only proper_email accepts * fix: judge instance group members by a mirror of the usr email constraint * fix: fold ascii only in the proper_email mirror, like the constraint * fix: let the database judge which instance group members usr will store * fix: cut a derived username to the column width so a long local part can be provisioned * chore: move the ee pin to the scim member doc fix * chore: update ee-repo-ref to 0780955effb657807d14f0eb503cba1d49cee007 This commit updates the EE repository reference after PR #801 was merged in windmill-ee-private. Previous ee-repo-ref: ee6452d489563204a98df883703f78d5e74cdd69 New ee-repo-ref: 0780955effb657807d14f0eb503cba1d49cee007 Automated by sync-ee-ref workflow. --------- Co-authored-by: windmill-internal-app[bot] --- backend/Cargo.lock | 1 + backend/ee-repo-ref.txt | 2 +- backend/windmill-api-groups/Cargo.toml | 1 + backend/windmill-api-groups/src/groups.rs | 24 ++- .../tests/groups.rs | 137 ++++++++++++++++++ backend/windmill-api-users/src/users.rs | 10 +- backend/windmill-common/src/usernames.rs | 37 ++++- backend/windmill-common/src/users.rs | 29 ++++ .../tests/usr_accepts_email.rs | 59 ++++++++ 9 files changed, 291 insertions(+), 9 deletions(-) create mode 100644 backend/windmill-common/tests/usr_accepts_email.rs diff --git a/backend/Cargo.lock b/backend/Cargo.lock index a1a358c4e4..873d3d0590 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -15253,6 +15253,7 @@ dependencies = [ "serde_json", "sql-builder", "sqlx", + "tracing", "uuid", "windmill-api-auth", "windmill-api-workspaces", diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 062211f925..00bf0cbe8f 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -1ba6fe83451f0a1f8fafe04b7187087d51e0f769 +0780955effb657807d14f0eb503cba1d49cee007 diff --git a/backend/windmill-api-groups/Cargo.toml b/backend/windmill-api-groups/Cargo.toml index 856eaacc0a..35443008f0 100644 --- a/backend/windmill-api-groups/Cargo.toml +++ b/backend/windmill-api-groups/Cargo.toml @@ -29,4 +29,5 @@ serde.workspace = true serde_json.workspace = true sql-builder.workspace = true sqlx.workspace = true +tracing.workspace = true uuid.workspace = true diff --git a/backend/windmill-api-groups/src/groups.rs b/backend/windmill-api-groups/src/groups.rs index a15b9823d3..c931cf2255 100644 --- a/backend/windmill-api-groups/src/groups.rs +++ b/backend/windmill-api-groups/src/groups.rs @@ -22,7 +22,10 @@ use windmill_common::{ error::{Error, JsonResult, Result}, utils::{not_found_if_none, paginate, Pagination}, }; -use windmill_common::{db::UserDB, users::username_to_permissioned_as}; +use windmill_common::{ + db::UserDB, + users::{username_to_permissioned_as, usr_accepts_email}, +}; use serde::{Deserialize, Serialize}; use sqlx::{query_scalar, FromRow, Postgres, Transaction}; @@ -972,6 +975,15 @@ async fn add_user_igroup( ) -> Result { require_super_admin(&db, &authed).await?; + // `email_to_igroup` has no shape constraint of its own; `usr`, which the member is + // promoted into on reconcile, has `proper_email`, and a value failing it there would + // roll back every member of the group. + if !usr_accepts_email(&db, &email).await? { + return Err(Error::BadRequest(format!( + "'{email}' is not a valid email address" + ))); + } + let mut tx: Transaction<'_, Postgres> = db.begin().await?; // FOR UPDATE: the group row is the group-level mutex, taken before the workspace @@ -1424,6 +1436,16 @@ async fn overwrite_igroups( if let Some(emails) = &igroup.emails { for email in emails.iter() { + // An export can carry a member the source instance stored before ingest + // validated member values; it is dropped rather than failing the import. + if !usr_accepts_email(&mut *tx, email).await? { + tracing::warn!( + "Skipping member '{}' of imported instance group '{}': not an email address", + email, + igroup.name + ); + continue; + } sqlx::query!( "INSERT INTO email_to_igroup (email, igroup) VALUES ($1, $2)", email, diff --git a/backend/windmill-api-integration-tests/tests/groups.rs b/backend/windmill-api-integration-tests/tests/groups.rs index f86522aae3..26e5caeea1 100644 --- a/backend/windmill-api-integration-tests/tests/groups.rs +++ b/backend/windmill-api-integration-tests/tests/groups.rs @@ -913,3 +913,140 @@ async fn test_preserve_orphaned_members_migration(db: Pool) -> anyhow: Ok(()) } + +/// A membership row whose value is not an email (an IdP object id a SCIM sync stored before +/// member values were validated) must not break the workspace's instance-group save: the +/// reconciler skips it and still provisions the valid members. The admin endpoint refuses to +/// add such a value in the first place. +#[cfg(feature = "private")] +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_instance_group_member_that_is_not_an_email_is_skipped( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let global_base = format!("http://localhost:{port}/api/groups"); + let ws_base = format!("http://localhost:{port}/api/w/test-workspace/workspaces"); + const ENTRA_OBJECT_ID: &str = "ef40ea04-1a9e-4a84-9e65-cb1baa81dfed"; + + let resp = authed(client().post(format!("{global_base}/create"))) + .json(&json!({ "name": "entra_grp" })) + .send() + .await?; + assert_eq!(resp.status(), 200, "create"); + let resp = authed(client().post(format!("{global_base}/adduser/entra_grp"))) + .json(&json!({ "email": "kept@example.com" })) + .send() + .await?; + assert_eq!(resp.status(), 200, "adduser"); + + let resp = authed(client().post(format!("{global_base}/adduser/entra_grp"))) + .json(&json!({ "email": ENTRA_OBJECT_ID })) + .send() + .await?; + assert_eq!( + resp.status(), + 400, + "adduser must refuse a value that is not an email" + ); + let too_wide = format!("{}@example.com", "a".repeat(244)); + let resp = authed(client().post(format!("{global_base}/adduser/entra_grp"))) + .json(&json!({ "email": too_wide })) + .send() + .await?; + assert_eq!( + resp.status(), + 400, + "adduser must refuse a value wider than the email columns" + ); + // A valid address whose local part is wider than the username columns: the derived + // username is cut to fit rather than failing the promotion. + let long_local_part = format!("{}@example.com", "a".repeat(60)); + let resp = authed(client().post(format!("{global_base}/adduser/entra_grp"))) + .json(&json!({ "email": long_local_part })) + .send() + .await?; + assert_eq!(resp.status(), 200, "adduser long local part"); + + sqlx::query("INSERT INTO email_to_igroup (email, igroup) VALUES ($1, 'entra_grp')") + .bind(ENTRA_OBJECT_ID) + .execute(&db) + .await?; + + // A member whose address only the wider `proper_email` of `usr` accepts, already + // provisioned through the group: reconciliation must keep and re-role them, since + // removal destroys their drafts, inputs and permissions. + sqlx::raw_sql( + r#" + INSERT INTO email_to_igroup (email, igroup) VALUES ('"quoted"@example.com', 'entra_grp'); + INSERT INTO usr (workspace_id, username, email, is_admin, operator, added_via) + VALUES ('test-workspace', 'quoted', '"quoted"@example.com', false, true, + '{"source": "instance_group", "group": "entra_grp"}'::jsonb); + "#, + ) + .execute(&db) + .await?; + + let resp = authed(client().post(format!("{ws_base}/edit_instance_groups"))) + .json(&json!({ + "groups": ["entra_grp"], + "roles": { "entra_grp": "developer" } + })) + .send() + .await?; + assert_eq!(resp.status(), 200, "edit: {}", resp.text().await?); + + let mut members: Vec<(String, bool)> = sqlx::query_as( + "SELECT email, operator FROM usr WHERE workspace_id = 'test-workspace' + AND added_via->>'source' = 'instance_group'", + ) + .fetch_all(&db) + .await?; + members.sort(); + assert_eq!( + members, + vec![ + ("\"quoted\"@example.com".to_string(), false), + (long_local_part.clone(), false), + ("kept@example.com".to_string(), false), + ], + "valid members provisioned and existing member kept, all as developers; non-email one skipped" + ); + + // A full import carrying the same rows: the object id is dropped, the address only + // `proper_email` accepts is kept, and neither member loses their workspace row. + let resp = authed(client().post(format!("{global_base}/overwrite"))) + .json(&json!([{ + "name": "entra_grp", + "emails": ["kept@example.com", "\"quoted\"@example.com", long_local_part, ENTRA_OBJECT_ID] + }])) + .send() + .await?; + assert_eq!(resp.status(), 200, "overwrite: {}", resp.text().await?); + + let mut stored: Vec = + sqlx::query_scalar("SELECT email FROM email_to_igroup WHERE igroup = 'entra_grp'") + .fetch_all(&db) + .await?; + stored.sort(); + assert_eq!( + stored, + vec![ + "\"quoted\"@example.com".to_string(), + long_local_part.clone(), + "kept@example.com".to_string(), + ], + "import drops the object id and keeps the rest" + ); + let mut after_import: Vec<(String, bool)> = sqlx::query_as( + "SELECT email, operator FROM usr WHERE workspace_id = 'test-workspace' + AND added_via->>'source' = 'instance_group'", + ) + .fetch_all(&db) + .await?; + after_import.sort(); + assert_eq!(after_import, members, "import must not evict either member"); + + Ok(()) +} diff --git a/backend/windmill-api-users/src/users.rs b/backend/windmill-api-users/src/users.rs index f152ba8808..476a900ef6 100644 --- a/backend/windmill-api-users/src/users.rs +++ b/backend/windmill-api-users/src/users.rs @@ -53,8 +53,8 @@ use windmill_common::per_minute_counter::PerMinuteCounter; use windmill_common::users::truncate_token; use windmill_common::users::COOKIE_NAME; use windmill_common::users::{ - username_to_permissioned_as, PERMISSIONED_AS_MAX_LEN, SUPERADMIN_NOTIFICATION_EMAIL, - SUPERADMIN_SECRET_EMAIL, SUPERADMIN_SYNC_EMAIL, VALID_EMAIL, + username_to_permissioned_as, EMAIL_COLUMN_MAX_LEN, PERMISSIONED_AS_MAX_LEN, + SUPERADMIN_NOTIFICATION_EMAIL, SUPERADMIN_SECRET_EMAIL, SUPERADMIN_SYNC_EMAIL, VALID_EMAIL, }; use windmill_common::utils::paginate; use windmill_common::worker::CLOUD_HOSTED; @@ -1758,7 +1758,6 @@ struct ChangeUserEmail { /// `varchar(50)`, and `v2_job.permissioned_as` in a `varchar(55)`; every other email column is /// `varchar(255)`. The strictest of the two bounds is used for all of them. const SHORT_EMAIL_COLUMN_MAX_LEN: usize = 50; -const EMAIL_COLUMN_MAX_LEN: usize = 255; /// Move an account to a new email address, in place: the `password` row (and with it the /// instance-wide username, the role and the login type) is kept and every email-keyed row is @@ -3253,7 +3252,10 @@ mod same_origin_rd_tests { /// Both provisioning writes reference `password(email)`; a typo'd address from the /// provisioning script should read as "no such account", not as a foreign-key error. -async fn require_account(tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, email: &str) -> Result<()> { +async fn require_account( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + email: &str, +) -> Result<()> { let exists = sqlx::query_scalar!( "SELECT EXISTS(SELECT 1 FROM password WHERE email = $1)", email diff --git a/backend/windmill-common/src/usernames.rs b/backend/windmill-common/src/usernames.rs index 6fde2a8186..adb81ae682 100644 --- a/backend/windmill-common/src/usernames.rs +++ b/backend/windmill-common/src/usernames.rs @@ -17,6 +17,23 @@ lazy_static::lazy_static! { pub static ref VALID_USERNAME: Regex = Regex::new(r#"^[a-zA-Z][a-zA-Z_0-9]*$"#).unwrap(); } +/// Width of the `username` columns of `usr`, `password` and `pending_user`. +pub const USERNAME_MAX_LEN: usize = 50; + +/// `base` with the collision suffix of `attempt` appended (none for the first attempt), cut +/// to `USERNAME_MAX_LEN`. A local part longer than the column is a valid email, and an +/// insert that fails on the derived username rolls back everything around it. +pub fn fit_username(base: &str, attempt: u32) -> String { + let suffix = if attempt > 1 { + attempt.to_string() + } else { + String::new() + }; + let mut username: String = base.chars().take(USERNAME_MAX_LEN - suffix.len()).collect(); + username.push_str(&suffix); + username +} + pub async fn generate_instance_wide_unique_username<'c>( tx: &mut Transaction<'c, Postgres>, email: &str, @@ -41,9 +58,7 @@ pub async fn generate_instance_wide_unique_username<'c>( email ))); } - if i > 1 { - username = format!("{}{}", base_username, i) - } + username = fit_username(&base_username, i); username_conflict = sqlx::query_scalar!( "SELECT EXISTS(SELECT 1 FROM usr WHERE username = $1 and email != $2 UNION SELECT 1 FROM password WHERE username = $1 UNION SELECT 1 FROM pending_user WHERE username = $1)", &username, @@ -164,3 +179,19 @@ pub async fn get_instance_username_or_create_pending<'c>( } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_fit_username_keeps_the_column_width() { + assert_eq!(fit_username("alice", 1), "alice"); + assert_eq!(fit_username("alice", 2), "alice2"); + let base = "a".repeat(60); + assert_eq!(fit_username(&base, 1), "a".repeat(USERNAME_MAX_LEN)); + let with_suffix = fit_username(&base, 1000); + assert_eq!(with_suffix.len(), USERNAME_MAX_LEN); + assert!(with_suffix.ends_with("1000")); + } +} diff --git a/backend/windmill-common/src/users.rs b/backend/windmill-common/src/users.rs index 941328a8b2..1b696746b0 100644 --- a/backend/windmill-common/src/users.rs +++ b/backend/windmill-common/src/users.rs @@ -13,6 +13,35 @@ lazy_static::lazy_static! { pub static ref VALID_EMAIL: regex::Regex = regex::Regex::new( r"^[A-Za-z0-9!#$%&'*+/=?^_`{|}~-]+(\.[A-Za-z0-9!#$%&'*+/=?^_`{|}~-]+)*@([A-Za-z0-9]([A-Za-z0-9-]*[A-Za-z0-9])?\.)+[A-Za-z0-9]([A-Za-z0-9-]*[A-Za-z0-9])?$" ).unwrap(); + +} + +/// Width of the `email` columns of `usr`, `workspace_invite` and `email_to_igroup`. +pub const EMAIL_COLUMN_MAX_LEN: usize = 255; + +/// The regex of the `proper_email` CHECK constraint on `usr` and `workspace_invite` +/// (`20220620210708_regex_fix`), verbatim, for [`usr_accepts_email`]. Evaluated by the +/// database and never by a Rust engine: `~*` folds case under the database collation, so a +/// fixed mirror accepts addresses the constraint rejects, or rejects ones it holds, on some +/// locale. `windmill-common/tests/usr_accepts_email.rs` pins the text to the constraint. +pub const PROPER_EMAIL_PATTERN: &str = r#"^(?:[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\[(?:(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9]))\.){3}(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9])|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\])$"#; + +/// Whether `usr` (and `workspace_invite`) will store `email`: the `proper_email` regex as the +/// database evaluates it, plus the column width. Unlike [`VALID_EMAIL`] this admits every +/// address those tables already hold, which matters wherever an existing member is judged. +pub async fn usr_accepts_email<'c, E>(db: E, email: &str) -> crate::error::Result +where + E: sqlx::Executor<'c, Database = sqlx::Postgres>, +{ + if email.contains('\0') || email.chars().count() > EMAIL_COLUMN_MAX_LEN { + return Ok(false); + } + let accepted: bool = sqlx::query_scalar("SELECT $1::text ~* $2::text") + .bind(email) + .bind(PROPER_EMAIL_PATTERN) + .fetch_one(db) + .await?; + Ok(accepted) } pub const SUPERADMIN_SECRET_EMAIL: &str = "superadmin_secret@windmill.dev"; diff --git a/backend/windmill-common/tests/usr_accepts_email.rs b/backend/windmill-common/tests/usr_accepts_email.rs new file mode 100644 index 0000000000..624d8812b4 --- /dev/null +++ b/backend/windmill-common/tests/usr_accepts_email.rs @@ -0,0 +1,59 @@ +//! `usr_accepts_email` predicts whether `usr` will store an address by evaluating +//! `PROPER_EMAIL_PATTERN` in the database. It only stays right while that text matches the +//! `proper_email` constraint and the width matches the column: each sample below must be +//! stored by `usr` exactly when the check accepts it, and everything `VALID_EMAIL` accepts +//! within the width must be stored too. + +use sqlx::{Pool, Postgres}; +use windmill_common::users::{usr_accepts_email, EMAIL_COLUMN_MAX_LEN, VALID_EMAIL}; + +#[sqlx::test(migrations = "../migrations")] +async fn usr_accepts_email_agrees_with_the_constraint(db: Pool) -> anyhow::Result<()> { + let domain = "@example.com"; + let widest = format!( + "{}{domain}", + "a".repeat(EMAIL_COLUMN_MAX_LEN - domain.len()) + ); + let too_wide = format!("a{widest}"); + for email in [ + "alice@example.com", + "Alice@Example.COM", + "alice.bob+tag@sub.example.co.uk", + "\"quoted\"@example.com", + "\"quoted local\"@example.com", + "alice@[192.168.0.1]", + widest.as_str(), + too_wide.as_str(), + "ef40ea04-1a9e-4a84-9e65-cb1baa81dfed", + // Unicode case folding would map the long s and the Kelvin sign into `[a-z]`. + "u\u{17f}er@example.com", + "alice@example\u{212a}.com", + "alice", + "alice@example", + "alice@@example.com", + "alice @example.com", + "alice@example.com\nbob@example.com", + "", + ] { + let mut tx = db.begin().await?; + let stored = sqlx::query( + "INSERT INTO usr (workspace_id, username, email, is_admin, operator) + VALUES ('admins', 'probe', $1, false, false)", + ) + .bind(email) + .execute(&mut *tx) + .await + .is_ok(); + tx.rollback().await?; + + assert_eq!( + stored, + usr_accepts_email(&db, email).await?, + "{email:?}: `usr` and usr_accepts_email disagree" + ); + if VALID_EMAIL.is_match(email) && email.len() <= EMAIL_COLUMN_MAX_LEN { + assert!(stored, "{email:?}: VALID_EMAIL accepts what `usr` rejects"); + } + } + Ok(()) +} From 96963080f1711192fe1d9bc4142c1710511504d4 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 15 Sep 2026 09:45:50 +0200 Subject: [PATCH 120/144] fix(python): parse wheel RECORD paths as RFC 4180 csv fields (#11133) Co-authored-by: Claude Fable 5.1 --- .../windmill-worker/src/python_executor.rs | 73 ++++++++++++++++++- 1 file changed, 69 insertions(+), 4 deletions(-) diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 175ed8295d..5f5ca170b4 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -2338,6 +2338,33 @@ async fn spawn_uv_install( } } +/// First field (the path) of a wheel RECORD line. RECORD is CSV (PEP 376 / +/// RFC 4180): a path containing a comma or a double quote is written quoted, +/// with inner quotes doubled, so splitting on the first comma turns such an +/// entry into a name that never exists on disk. +fn record_first_field(line: &str) -> Option { + let Some(quoted) = line.strip_prefix('"') else { + return line + .split(',') + .next() + .filter(|p| !p.is_empty()) + .map(str::to_owned); + }; + let mut field = String::new(); + let mut chars = quoted.chars(); + while let Some(c) = chars.next() { + if c != '"' { + field.push(c); + } else if chars.as_str().starts_with('"') { + chars.next(); + field.push('"'); + } else { + return Some(field).filter(|f| !f.is_empty()); + } + } + None +} + /// Verify that every file listed in the wheel's RECORD exists on disk under /// `venv_p`. Used as a structural integrity check after both a successful /// `pull_from_tar` (object-store cache hit) and a successful local @@ -2386,9 +2413,9 @@ async fn verify_wheel_record(venv_p: &str) -> Result<(), String> { if trimmed.is_empty() { continue; } - let rel_path = match trimmed.split(',').next() { - Some(p) if !p.is_empty() => p, - _ => continue, + let rel_path = match record_first_field(trimmed) { + Some(p) => p, + None => continue, }; // Defensive: skip absolute paths or escaping entries — we only // validate package-relative files. @@ -2397,7 +2424,7 @@ async fn verify_wheel_record(venv_p: &str) -> Result<(), String> { } let full = format!("{venv_p}/{rel_path}"); if tokio::fs::metadata(&full).await.is_err() { - missing.push(rel_path.to_string()); + missing.push(rel_path); // Bound error size in pathological cases (e.g. wholly empty dir). if missing.len() >= 10 { missing.push("...".to_string()); @@ -3752,6 +3779,44 @@ mod tests { .is_ok()); } + #[tokio::test] + async fn test_verify_wheel_record_accepts_csv_quoted_path() { + let dir = tempfile::tempdir().unwrap(); + // A path containing a comma is CSV-quoted in RECORD (wcwidth 0.8.3 + // ships `wcwidth/textwrap.py,cover`). Splitting on the first comma + // looked for `"pkg/textwrap.py` and rejected a complete install. + write_fake_wheel( + dir.path(), + &["pkg/textwrap.py", "pkg/textwrap.py,cover"], + &[ + "pkg/textwrap.py,sha256=aaa,1", + "\"pkg/textwrap.py,cover\",sha256=bbb,1", + "pkg-1.0.0.dist-info/RECORD,,", + ], + ); + assert!(verify_wheel_record(dir.path().to_str().unwrap()) + .await + .is_ok()); + } + + #[test] + fn test_record_first_field_unquotes_rfc4180() { + assert_eq!( + record_first_field("pkg/a.py,sha256=x,1").as_deref(), + Some("pkg/a.py") + ); + assert_eq!( + record_first_field("\"pkg/a.py,cover\",sha256=x,1").as_deref(), + Some("pkg/a.py,cover") + ); + assert_eq!( + record_first_field("\"pkg/say \"\"hi\"\".py\",sha256=x,1").as_deref(), + Some("pkg/say \"hi\".py") + ); + assert_eq!(record_first_field(",,"), None); + assert_eq!(record_first_field("\"unterminated,sha256=x,1"), None); + } + // Regression tests for the concurrent-install guard. Two jobs installing the // same uncached dep into the shared `venv_p` used to race uv's `--reinstall`, // corrupting the on-disk wheel and failing with "Env installation did not From 082d8973282690d82b54af4670d5aec67c0de217 Mon Sep 17 00:00:00 2001 From: Alexander Petric Date: Tue, 15 Sep 2026 04:30:19 -0400 Subject: [PATCH 121/144] stop the variables page opening a drawer for the instance settings hash (#11129) Claude-Session: https://claude.ai/code/session_01LCt167dnWJ2EVnLpkHATh4 Co-authored-by: Claude Opus 5 (1M context) --- frontend/src/routes/(root)/(logged)/variables/+page.svelte | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frontend/src/routes/(root)/(logged)/variables/+page.svelte b/frontend/src/routes/(root)/(logged)/variables/+page.svelte index 52c248ea3e..873e812987 100644 --- a/frontend/src/routes/(root)/(logged)/variables/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/variables/+page.svelte @@ -264,7 +264,9 @@ let handledHash = '' $effect(() => { const hash = $page.url.hash - if (hash.length <= 1) { + // Only item paths are drawer targets: the same hash also carries global + // drawers like #superadmin-settings, which must not be looked up as a variable. + if (!/^#[ufg]\//.test(hash)) { // Navigating away from a drawer target must clear the tracker, or // re-targeting the same item later would be skipped as already handled. handledHash = '' From 69e6efd875e020779ea115c096331da8eae2ffe7 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Tue, 15 Sep 2026 10:34:57 +0200 Subject: [PATCH 122/144] fix(git-sync): run auto-pull as the admin who enabled it (#11121) * fix(git-sync): run auto-pull as the admin who enabled it Co-Authored-By: Claude Opus 5 * fix(git-sync): audit the admin grant fork pulls make Co-Authored-By: Claude Opus 5 * chore: bump ee ref for the post-commit fork grant audit Co-Authored-By: Claude Opus 5 * fix(git-sync): address review nits on the auto-pull stamp Co-Authored-By: Claude Opus 5 * chore: update ee-repo-ref to ccada062c072d7b74894b63863728fd1ef9bdffd This commit updates the EE repository reference after PR #799 was merged in windmill-ee-private. Previous ee-repo-ref: 7cee30f0cf12721cba551cd754dc817444810470 New ee-repo-ref: ccada062c072d7b74894b63863728fd1ef9bdffd Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Opus 5 Co-authored-by: windmill-internal-app[bot] --- ...e4e0b12b105d4475d7eba2d3a9573b93e388.json} | 4 +- ...246fd2bdf9e3fce89852120aa4ad4ad6abfc3.json | 26 --- ...0a7b681d101d286adfc6ff4548d1b9fdbab8c.json | 22 ++ ...dfec43dc8a042f9d95e63bf84b2dc15a72165.json | 23 +++ ...75f8d2f1665d329b8adea424b3a6f1e5c7013.json | 47 +++++ ...0baab26b8160929aea6d8f5f226e6ec4f8bd8.json | 24 +++ ...347e22196e46727487f35634af079c71c2bef.json | 23 +++ backend/Cargo.lock | 1 + backend/ee-repo-ref.txt | 2 +- .../fixtures/git_sync_autopull_identity.sql | 46 +++++ backend/tests/git_sync_autopull_identity.rs | 194 ++++++++++++++++++ .../tests/workspace_dependencies_git_sync.rs | 1 + .../windmill-api-workspaces/src/workspaces.rs | 39 +++- backend/windmill-api/openapi.yaml | 3 + backend/windmill-api/src/workspaces_export.rs | 9 +- backend/windmill-common/src/workspaces.rs | 7 + backend/windmill-git-sync/Cargo.toml | 5 +- docs/git-sync-pull-design.md | 29 +++ .../git_sync/GitSyncContext.svelte.ts | 11 +- .../git_sync/GitSyncRepositoryCard.svelte | 8 + 20 files changed, 481 insertions(+), 43 deletions(-) rename backend/.sqlx/{query-3202bed875693ae923f496272cd8ad89b2f17a9d3ef4659c2d2284415177b32c.json => query-0c4dc0e9dc159fac7e41492c78a4e4e0b12b105d4475d7eba2d3a9573b93e388.json} (54%) delete mode 100644 backend/.sqlx/query-17cdf02b4912078459526205849246fd2bdf9e3fce89852120aa4ad4ad6abfc3.json create mode 100644 backend/.sqlx/query-3fe41e2a72d02613a2b1c1c44fb0a7b681d101d286adfc6ff4548d1b9fdbab8c.json create mode 100644 backend/.sqlx/query-6dc8032100a28c4a6e843370038dfec43dc8a042f9d95e63bf84b2dc15a72165.json create mode 100644 backend/.sqlx/query-8bdfc02e7be54c2b610fed11cce75f8d2f1665d329b8adea424b3a6f1e5c7013.json create mode 100644 backend/.sqlx/query-a80a18774baf36d09b07da1e4e30baab26b8160929aea6d8f5f226e6ec4f8bd8.json create mode 100644 backend/.sqlx/query-b675c20bb7a15bec5e9a34d7ddf347e22196e46727487f35634af079c71c2bef.json create mode 100644 backend/tests/fixtures/git_sync_autopull_identity.sql create mode 100644 backend/tests/git_sync_autopull_identity.rs diff --git a/backend/.sqlx/query-3202bed875693ae923f496272cd8ad89b2f17a9d3ef4659c2d2284415177b32c.json b/backend/.sqlx/query-0c4dc0e9dc159fac7e41492c78a4e4e0b12b105d4475d7eba2d3a9573b93e388.json similarity index 54% rename from backend/.sqlx/query-3202bed875693ae923f496272cd8ad89b2f17a9d3ef4659c2d2284415177b32c.json rename to backend/.sqlx/query-0c4dc0e9dc159fac7e41492c78a4e4e0b12b105d4475d7eba2d3a9573b93e388.json index 0a833ba620..446c785770 100644 --- a/backend/.sqlx/query-3202bed875693ae923f496272cd8ad89b2f17a9d3ef4659c2d2284415177b32c.json +++ b/backend/.sqlx/query-0c4dc0e9dc159fac7e41492c78a4e4e0b12b105d4475d7eba2d3a9573b93e388.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT username, email FROM usr WHERE workspace_id = $1 AND is_admin = true AND operator = false AND disabled = false ORDER BY username LIMIT 1", + "query": "SELECT u.username, u.email FROM usr u WHERE u.workspace_id = $1 AND u.is_admin AND NOT u.operator AND NOT u.disabled AND NOT EXISTS (SELECT 1 FROM password p WHERE p.email = u.email AND p.disabled) ORDER BY u.username LIMIT 1", "describe": { "columns": [ { @@ -24,5 +24,5 @@ false ] }, - "hash": "3202bed875693ae923f496272cd8ad89b2f17a9d3ef4659c2d2284415177b32c" + "hash": "0c4dc0e9dc159fac7e41492c78a4e4e0b12b105d4475d7eba2d3a9573b93e388" } diff --git a/backend/.sqlx/query-17cdf02b4912078459526205849246fd2bdf9e3fce89852120aa4ad4ad6abfc3.json b/backend/.sqlx/query-17cdf02b4912078459526205849246fd2bdf9e3fce89852120aa4ad4ad6abfc3.json deleted file mode 100644 index bc2e6a6f63..0000000000 --- a/backend/.sqlx/query-17cdf02b4912078459526205849246fd2bdf9e3fce89852120aa4ad4ad6abfc3.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT COALESCE(username, split_part(email, '@', 1)) AS \"username!\", email FROM password WHERE super_admin = true AND disabled = false ORDER BY email LIMIT 1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "username!", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "email", - "type_info": "Varchar" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - null, - false - ] - }, - "hash": "17cdf02b4912078459526205849246fd2bdf9e3fce89852120aa4ad4ad6abfc3" -} diff --git a/backend/.sqlx/query-3fe41e2a72d02613a2b1c1c44fb0a7b681d101d286adfc6ff4548d1b9fdbab8c.json b/backend/.sqlx/query-3fe41e2a72d02613a2b1c1c44fb0a7b681d101d286adfc6ff4548d1b9fdbab8c.json new file mode 100644 index 0000000000..564c784d22 --- /dev/null +++ b/backend/.sqlx/query-3fe41e2a72d02613a2b1c1c44fb0a7b681d101d286adfc6ff4548d1b9fdbab8c.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT super_admin AS \"super_admin!\" FROM password WHERE email = $1 AND disabled = false", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "super_admin!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "3fe41e2a72d02613a2b1c1c44fb0a7b681d101d286adfc6ff4548d1b9fdbab8c" +} diff --git a/backend/.sqlx/query-6dc8032100a28c4a6e843370038dfec43dc8a042f9d95e63bf84b2dc15a72165.json b/backend/.sqlx/query-6dc8032100a28c4a6e843370038dfec43dc8a042f9d95e63bf84b2dc15a72165.json new file mode 100644 index 0000000000..8026b07416 --- /dev/null +++ b/backend/.sqlx/query-6dc8032100a28c4a6e843370038dfec43dc8a042f9d95e63bf84b2dc15a72165.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT r->'auto_pull'->>'enabled_by' AS \"enabled_by\"\n FROM workspace_settings, jsonb_array_elements(git_sync->'repositories') r\n WHERE workspace_id = $1 AND r->>'git_repo_resource_path' = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "enabled_by", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "6dc8032100a28c4a6e843370038dfec43dc8a042f9d95e63bf84b2dc15a72165" +} diff --git a/backend/.sqlx/query-8bdfc02e7be54c2b610fed11cce75f8d2f1665d329b8adea424b3a6f1e5c7013.json b/backend/.sqlx/query-8bdfc02e7be54c2b610fed11cce75f8d2f1665d329b8adea424b3a6f1e5c7013.json new file mode 100644 index 0000000000..0bcfd38acd --- /dev/null +++ b/backend/.sqlx/query-8bdfc02e7be54c2b610fed11cce75f8d2f1665d329b8adea424b3a6f1e5c7013.json @@ -0,0 +1,47 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT u.username, u.is_admin, u.operator, u.disabled, EXISTS (SELECT 1 FROM password p WHERE p.email = u.email AND p.disabled) AS \"instance_disabled!\" FROM usr u WHERE u.workspace_id = $1 AND u.email = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "username", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "is_admin", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "operator", + "type_info": "Bool" + }, + { + "ordinal": 3, + "name": "disabled", + "type_info": "Bool" + }, + { + "ordinal": 4, + "name": "instance_disabled!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false, + null + ] + }, + "hash": "8bdfc02e7be54c2b610fed11cce75f8d2f1665d329b8adea424b3a6f1e5c7013" +} diff --git a/backend/.sqlx/query-a80a18774baf36d09b07da1e4e30baab26b8160929aea6d8f5f226e6ec4f8bd8.json b/backend/.sqlx/query-a80a18774baf36d09b07da1e4e30baab26b8160929aea6d8f5f226e6ec4f8bd8.json new file mode 100644 index 0000000000..bc12fdba1f --- /dev/null +++ b/backend/.sqlx/query-a80a18774baf36d09b07da1e4e30baab26b8160929aea6d8f5f226e6ec4f8bd8.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO usr (workspace_id, username, email, is_admin, operator)\n SELECT $1::varchar, u.username, u.email, true, false FROM usr u\n WHERE u.workspace_id = $2 AND u.email = $3 AND u.is_admin AND NOT u.operator AND NOT u.disabled\n AND NOT EXISTS (SELECT 1 FROM password p WHERE p.email = u.email AND p.disabled)\n AND NOT EXISTS (SELECT 1 FROM usr f WHERE f.workspace_id = $1::varchar AND f.email = $3)\n ON CONFLICT DO NOTHING\n RETURNING username", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "username", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "a80a18774baf36d09b07da1e4e30baab26b8160929aea6d8f5f226e6ec4f8bd8" +} diff --git a/backend/.sqlx/query-b675c20bb7a15bec5e9a34d7ddf347e22196e46727487f35634af079c71c2bef.json b/backend/.sqlx/query-b675c20bb7a15bec5e9a34d7ddf347e22196e46727487f35634af079c71c2bef.json new file mode 100644 index 0000000000..f67a0f39f2 --- /dev/null +++ b/backend/.sqlx/query-b675c20bb7a15bec5e9a34d7ddf347e22196e46727487f35634af079c71c2bef.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS (SELECT 1 FROM usr WHERE workspace_id = $1 AND username = $2) AS \"claimed!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "claimed!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "b675c20bb7a15bec5e9a34d7ddf347e22196e46727487f35634af079c71c2bef" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 873d3d0590..3d1921ee76 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -15727,6 +15727,7 @@ dependencies = [ "tokio", "tracing", "uuid", + "windmill-audit", "windmill-common", "windmill-dep-map", "windmill-queue", diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 00bf0cbe8f..461156c173 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -0780955effb657807d14f0eb503cba1d49cee007 +ccada062c072d7b74894b63863728fd1ef9bdffd diff --git a/backend/tests/fixtures/git_sync_autopull_identity.sql b/backend/tests/fixtures/git_sync_autopull_identity.sql new file mode 100644 index 0000000000..3ae7d03d0e --- /dev/null +++ b/backend/tests/fixtures/git_sync_autopull_identity.sql @@ -0,0 +1,46 @@ +-- A parent workspace whose auto-pulled repository was last saved by alice, and a fork of +-- it holding only carol, the non-admin who created it. aaron is an admin who sorts before +-- alice; bob is no longer an admin; dora is a workspace admin deactivated on the instance. +-- sam and sue are instance superadmins who are not members: sam's instance username is +-- carol's, sue's is unclaimed. + +INSERT INTO workspace (id, name, owner) VALUES ('ap-parent', 'ap-parent', 'alice@windmill.dev'); +INSERT INTO workspace (id, name, owner, parent_workspace_id) + VALUES ('wm-fork-feat', 'feat', 'carol@windmill.dev', 'ap-parent'); + +INSERT INTO group_ (workspace_id, name, summary, extra_perms) VALUES + ('ap-parent', 'all', 'All users', '{}'), + ('wm-fork-feat', 'all', 'All users', '{}'); + +INSERT INTO password (email, password_hash, login_type, super_admin, verified, name, disabled) VALUES + ('aaron@windmill.dev', 'not-a-real-hash', 'password', false, true, 'aaron', false), + ('alice@windmill.dev', 'not-a-real-hash', 'password', false, true, 'alice', false), + ('bob@windmill.dev', 'not-a-real-hash', 'password', false, true, 'bob', false), + ('carol@windmill.dev', 'not-a-real-hash', 'password', false, true, 'carol', false), + ('dora@windmill.dev', 'not-a-real-hash', 'password', false, true, 'dora', true); + +INSERT INTO password (email, password_hash, login_type, super_admin, verified, name, disabled, username) VALUES + ('sam@windmill.dev', 'not-a-real-hash', 'password', true, true, 'sam', false, 'carol'), + ('sue@windmill.dev', 'not-a-real-hash', 'password', true, true, 'sue', false, 'sue'); + +INSERT INTO usr (workspace_id, email, username, is_admin) VALUES + ('ap-parent', 'aaron@windmill.dev', 'aaron', true), + ('ap-parent', 'alice@windmill.dev', 'alice', true), + ('ap-parent', 'bob@windmill.dev', 'bob', false), + ('ap-parent', 'carol@windmill.dev', 'carol', false), + ('ap-parent', 'dora@windmill.dev', 'dora', true), + ('wm-fork-feat', 'carol@windmill.dev', 'carol', false); + +INSERT INTO resource (workspace_id, path, value, resource_type, extra_perms, created_by) VALUES + ('ap-parent', 'u/alice/repo', '{"url": "https://github.com/test/repo.git", "branch": "main"}', + 'git_repository', '{}', 'alice'), + ('wm-fork-feat', 'u/alice/repo', '{"url": "https://github.com/test/repo.git", "branch": "main"}', + 'git_repository', '{}', 'alice'); + +INSERT INTO workspace_settings (workspace_id, git_sync) VALUES + ('ap-parent', '{"repositories":[{"git_repo_resource_path":"$res:u/alice/repo", + "use_individual_branch":false,"group_by_folder":false, + "auto_pull":{"enabled":true,"mode":"polling","sync_forks":true, + "enabled_by":"alice@windmill.dev"}}]}'), + ('wm-fork-feat', '{"repositories":[{"git_repo_resource_path":"$res:u/alice/repo", + "use_individual_branch":false,"group_by_folder":false}]}'); diff --git a/backend/tests/git_sync_autopull_identity.rs b/backend/tests/git_sync_autopull_identity.rs new file mode 100644 index 0000000000..4e7562c3f3 --- /dev/null +++ b/backend/tests/git_sync_autopull_identity.rs @@ -0,0 +1,194 @@ +//! An automatic pull runs as the admin stamped on the repository's settings, never as +//! someone picked from the workspace, and stops once that admin is revoked. A fork's +//! pull runs as the parent's pull identity, added to the fork first. +#![cfg(all(feature = "enterprise", feature = "private"))] + +use sqlx::{Pool, Postgres}; +use windmill_common::workspaces::GitRepositorySettings; +use windmill_git_sync::{reconcile_and_enqueue_pull, reconcile_fork_branch_pull}; + +const PARENT: &str = "ap-parent"; +const FORK: &str = "wm-fork-feat"; +const REPO: &str = "$res:u/alice/repo"; + +fn repo_enabled_by(email: &str) -> GitRepositorySettings { + serde_json::from_value(serde_json::json!({ + "git_repo_resource_path": REPO, + "use_individual_branch": false, + "group_by_folder": false, + "auto_pull": { "enabled": true, "enabled_by": email } + })) + .expect("repository settings") +} + +/// `(created_by, permissioned_as, permissioned_as_email)` of every pull job in `w_id`. +async fn pull_identities( + db: &Pool, + w_id: &str, +) -> anyhow::Result)>> { + Ok(sqlx::query_as( + "SELECT created_by, permissioned_as, permissioned_as_email FROM v2_job \ + WHERE workspace_id = $1 AND kind = 'deploymentcallback'", + ) + .bind(w_id) + .fetch_all(db) + .await?) +} + +fn identity(username: &str) -> (String, String, Option) { + ( + username.to_string(), + format!("u/{username}"), + Some(format!("{username}@windmill.dev")), + ) +} + +async fn recorded_pull_error(db: &Pool, w_id: &str) -> anyhow::Result { + let git_sync: serde_json::Value = + sqlx::query_scalar("SELECT git_sync FROM workspace_settings WHERE workspace_id = $1") + .bind(w_id) + .fetch_one(db) + .await?; + Ok( + git_sync["repositories"][0]["auto_pull"]["last_pull_status"]["error"] + .as_str() + .unwrap_or_default() + .to_string(), + ) +} + +#[sqlx::test(fixtures("git_sync_autopull_identity"))] +async fn pull_runs_as_the_admin_who_enabled_it(db: Pool) -> anyhow::Result<()> { + let job = reconcile_and_enqueue_pull( + &db, + PARENT, + &repo_enabled_by("alice@windmill.dev"), + "main", + "abc123", + None, + ) + .await?; + + assert!(job.is_some()); + assert_eq!(pull_identities(&db, PARENT).await?, vec![identity("alice")]); + Ok(()) +} + +/// bob was demoted in the workspace; dora is still a workspace admin but deactivated on +/// the instance. Neither may run the pull, and the failure lands on the status rather +/// than as an error, which a webhook delivery would turn into a failed response. +#[sqlx::test(fixtures("git_sync_autopull_identity"))] +async fn pull_fails_on_the_status_once_the_enabling_admin_is_revoked( + db: Pool, +) -> anyhow::Result<()> { + for email in ["bob@windmill.dev", "dora@windmill.dev"] { + let job = reconcile_and_enqueue_pull( + &db, + PARENT, + &repo_enabled_by(email), + "main", + "abc123", + None, + ) + .await?; + assert!(job.is_none(), "{email} must not run the pull"); + let error = recorded_pull_error(&db, PARENT).await?; + assert!(error.contains(email), "{error}"); + } + assert!(pull_identities(&db, PARENT).await?.is_empty()); + Ok(()) +} + +#[sqlx::test(fixtures("git_sync_autopull_identity"))] +async fn fork_pull_runs_as_the_parent_admin_added_to_the_fork( + db: Pool, +) -> anyhow::Result<()> { + let job = reconcile_fork_branch_pull(&db, PARENT, REPO, "wm-fork/main/feat", "main", "abc123") + .await?; + assert!( + job.is_some(), + "the fork branch must route to the fork and enqueue" + ); + + let (is_admin, in_all): (bool, bool) = sqlx::query_as( + "SELECT u.is_admin, EXISTS (SELECT 1 FROM usr_to_group g \ + WHERE g.workspace_id = u.workspace_id AND g.usr = u.username AND g.group_ = 'all') \ + FROM usr u WHERE u.workspace_id = $1 AND u.email = 'alice@windmill.dev'", + ) + .bind(FORK) + .fetch_one(&db) + .await?; + assert!( + is_admin && in_all, + "alice must be an admin member of the fork" + ); + assert_eq!(pull_identities(&db, FORK).await?, vec![identity("alice")]); + + let grants: i64 = sqlx::query_scalar( + "SELECT count(*) FROM audit_partitioned WHERE workspace_id = $1 \ + AND operation = 'users.git_sync_fork_add' AND resource = 'alice@windmill.dev'", + ) + .bind(FORK) + .fetch_one(&db) + .await?; + assert_eq!(grants, 1, "adding alice to the fork must be audited"); + Ok(()) +} + +/// A superadmin who is not a member runs the pull under their instance username, and +/// `u/` resolves through the workspace's members first. sam's instance username +/// is carol's, so sam's stamp must not run the pull as carol; sue's is unclaimed. +#[sqlx::test(fixtures("git_sync_autopull_identity"))] +async fn a_non_member_superadmin_runs_the_pull_only_under_an_unclaimed_username( + db: Pool, +) -> anyhow::Result<()> { + let job = reconcile_and_enqueue_pull( + &db, + PARENT, + &repo_enabled_by("sam@windmill.dev"), + "main", + "abc123", + None, + ) + .await?; + assert!(job.is_none(), "sam's username belongs to carol"); + assert!(pull_identities(&db, PARENT).await?.is_empty()); + + let job = reconcile_and_enqueue_pull( + &db, + PARENT, + &repo_enabled_by("sue@windmill.dev"), + "main", + "abc123", + None, + ) + .await?; + assert!(job.is_some()); + assert_eq!(pull_identities(&db, PARENT).await?, vec![identity("sue")]); + Ok(()) +} + +/// With no stamp on the parent, the fork pull still runs as the parent's first active +/// admin: the fork holds only its non-admin creator, so no identity resolved in the fork +/// could run it. +#[sqlx::test(fixtures("git_sync_autopull_identity"))] +async fn unstamped_fork_pull_runs_as_the_parents_first_admin( + db: Pool, +) -> anyhow::Result<()> { + sqlx::query( + "UPDATE workspace_settings SET git_sync = git_sync #- '{repositories,0,auto_pull,enabled_by}' \ + WHERE workspace_id = $1", + ) + .bind(PARENT) + .execute(&db) + .await?; + + let job = reconcile_fork_branch_pull(&db, PARENT, REPO, "wm-fork/main/feat", "main", "abc123") + .await?; + assert!( + job.is_some(), + "an unstamped parent must still sync its forks" + ); + assert_eq!(pull_identities(&db, FORK).await?, vec![identity("aaron")]); + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/workspace_dependencies_git_sync.rs b/backend/windmill-api-integration-tests/tests/workspace_dependencies_git_sync.rs index 4e25be1fa5..dc8e8dcc00 100644 --- a/backend/windmill-api-integration-tests/tests/workspace_dependencies_git_sync.rs +++ b/backend/windmill-api-integration-tests/tests/workspace_dependencies_git_sync.rs @@ -923,6 +923,7 @@ async fn test_pull_stays_on_the_workspace_lane(db: Pool) -> anyhow::Re &db, "test-workspace", &repo, + ("test-user", "test@windmill.dev"), None, false, None, diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 3a2b1d715a..640c0fe06a 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -910,12 +910,15 @@ fn redact_git_sync_webhook_secrets(git_sync: &mut serde_json::Value) { } /// Zero the server-owned auto-pull fields (webhook id/secret/url/error, synced -/// sha, last pull status) on a client-supplied `AutoPullSettings`. The client only -/// controls `enabled` / `mode` / `poll_interval_s`; the rest is written by the -/// server (webhook creation, poller) and must never be trusted from the request — -/// otherwise a caller could inject a webhook id/secret or fake sync state. -fn clear_client_supplied_auto_pull_state( +/// sha, last pull status) on a client-supplied `AutoPullSettings`, and stamp who +/// pulls run as: `saver_email` while auto pull is on. The client only controls +/// `enabled` / `mode` / `poll_interval_s`; the rest is written by the server (webhook +/// creation, poller, this save) and must never be trusted from the request — +/// otherwise a caller could inject a webhook id/secret, fake sync state, or pick who +/// pulls run as. +fn sanitize_client_auto_pull( auto_pull: &mut windmill_common::workspaces::AutoPullSettings, + saver_email: &str, ) { auto_pull.webhook_id = None; auto_pull.webhook_secret = None; @@ -923,6 +926,28 @@ fn clear_client_supplied_auto_pull_state( auto_pull.webhook_error = None; auto_pull.last_synced_sha = std::collections::HashMap::new(); auto_pull.last_pull_status = None; + auto_pull.enabled_by = auto_pull.enabled.then(|| saver_email.to_string()); +} + +#[cfg(test)] +mod sanitize_client_auto_pull_tests { + use windmill_common::workspaces::AutoPullSettings; + + #[test] + fn a_save_stamps_the_saver_over_any_client_supplied_stamp() { + let mut ap = AutoPullSettings { + enabled: true, + enabled_by: Some("forged@example.com".to_string()), + ..Default::default() + }; + super::sanitize_client_auto_pull(&mut ap, "saver@example.com"); + assert_eq!(ap.enabled_by.as_deref(), Some("saver@example.com")); + + ap.enabled = false; + ap.enabled_by = Some("forged@example.com".to_string()); + super::sanitize_client_auto_pull(&mut ap, "saver@example.com"); + assert_eq!(ap.enabled_by, None, "auto pull off carries no stamp"); + } } /// Whether a git-sync repository tracking `tracked` rules out `label_branch` as a dev workspace's @@ -3983,7 +4008,7 @@ async fn edit_git_sync_config( // stay clean. for repo in git_sync_settings.repositories.iter_mut() { if let Some(ap) = repo.auto_pull.as_mut() { - clear_client_supplied_auto_pull_state(ap); + sanitize_client_auto_pull(ap, &authed.email); } repo.open_pr_error = None; repo.credential = None; @@ -4230,7 +4255,7 @@ async fn edit_git_sync_repository( // existing repo re-derives it from the DB (carried over below) and a new one // starts clean. if let Some(ap) = new_config.repository.auto_pull.as_mut() { - clear_client_supplied_auto_pull_state(ap); + sanitize_client_auto_pull(ap, &authed.email); } new_config.repository.open_pr_error = None; new_config.repository.credential = None; diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 411d5f658c..b733157d85 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -35418,6 +35418,9 @@ components: type: string last_pull_status: $ref: "#/components/schemas/AutoPullStatus" + enabled_by: + type: string + description: Email of the admin automatic pulls apply changes as. Set by the server when the settings are saved. required: - enabled diff --git a/backend/windmill-api/src/workspaces_export.rs b/backend/windmill-api/src/workspaces_export.rs index 828509519e..a85f9d004e 100644 --- a/backend/windmill-api/src/workspaces_export.rs +++ b/backend/windmill-api/src/workspaces_export.rs @@ -1587,10 +1587,10 @@ pub(crate) async fn tarball_workspace( // Use v2 format only if explicitly requested, otherwise use v1 (legacy) for backward compatibility // Server-owned state (the HMAC webhook secret + hook id/error, the - // synced-sha / last-pull status, and what the credential check observed) - // must never leave the server: keep it out of export archives and synced - // repos, and don't let a re-imported workspace inherit another install's - // hook/sync state. Mirrors the GET-settings redaction. + // synced-sha / last-pull status, the admin automatic pulls run as, and what + // the credential check observed) must never leave the server: keep it out of + // export archives and synced repos, and don't let a re-imported workspace + // inherit another install's hook/sync state or pull identity. fn redact_git_sync_for_export(git_sync: Option) -> Option { let mut git_sync = git_sync?; if let Some(repos) = git_sync @@ -1607,6 +1607,7 @@ pub(crate) async fn tarball_workspace( "webhook_error", "last_synced_sha", "last_pull_status", + "enabled_by", ] { auto_pull.remove(field); } diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index 3b00a87920..9b6a471b2b 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -514,6 +514,11 @@ pub struct AutoPullSettings { pub last_synced_sha: std::collections::HashMap, #[serde(default, skip_serializing_if = "Option::is_none")] pub last_pull_status: Option, + /// Email of the admin this repository's automatic pulls (its own and its forks') + /// apply changes as: whoever last saved the settings with auto pull on. Stamped + /// server-side, never taken from the client. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub enabled_by: Option, } // Manual Debug so the HMAC `webhook_secret` (even encrypted) never lands in logs. @@ -524,6 +529,7 @@ impl std::fmt::Debug for AutoPullSettings { .field("mode", &self.mode) .field("poll_interval_s", &self.poll_interval_s) .field("sync_forks", &self.sync_forks) + .field("enabled_by", &self.enabled_by) .field("webhook_id", &self.webhook_id) .field( "webhook_secret", @@ -2709,6 +2715,7 @@ mod tests { webhook_secret: None, webhook_url: None, webhook_error: None, + enabled_by: None, last_synced_sha: synced .iter() .map(|(r, s)| (r.to_string(), s.to_string())) diff --git a/backend/windmill-git-sync/Cargo.toml b/backend/windmill-git-sync/Cargo.toml index f74581ba67..2b691ee48e 100644 --- a/backend/windmill-git-sync/Cargo.toml +++ b/backend/windmill-git-sync/Cargo.toml @@ -9,8 +9,8 @@ name = "windmill_git_sync" path = "./src/lib.rs" [features] -private = ["windmill-common/private", "windmill-dep-map/private"] -enterprise = ["windmill-queue/enterprise", "windmill-common/enterprise"] +private = ["windmill-common/private", "windmill-dep-map/private", "windmill-audit/private"] +enterprise = ["windmill-queue/enterprise", "windmill-common/enterprise", "windmill-audit/enterprise"] all_sqlx_features = ["enterprise"] default = [] @@ -23,5 +23,6 @@ tracing.workspace = true windmill-common = { workspace = true, default-features = false } windmill-queue.workspace = true windmill-dep-map.workspace = true +windmill-audit.workspace = true regex = "1.10.3" tokio = { workspace = true, features = ["full"] } \ No newline at end of file diff --git a/docs/git-sync-pull-design.md b/docs/git-sync-pull-design.md index 5f77a3f117..53d36c82fe 100644 --- a/docs/git-sync-pull-design.md +++ b/docs/git-sync-pull-design.md @@ -186,6 +186,35 @@ Routing — an event/poll result is `(repo, ref, head_sha, sender)`: filters): fan out, each workspace pulls with its own filters; `wmill.yaml` in the repo stays authoritative for include/exclude. +Identity — a pull applies changes as a real workspace admin, never a reserved identity. +The schedules, triggers and app policies it deploys persist their deployer as the identity +they run as, and `validate_on_behalf_of` refuses reserved sentinels there, so a real admin +is what keeps those deployable and revocable (demote or remove the admin and what runs +under them stops). + +- The admin is `auto_pull.enabled_by`, stamped server-side with the email of whoever last + saved the git sync settings with auto pull on. Re-saving as another admin rotates it. +- A stamp naming someone who is no longer an active admin (demoted, or deactivated in the + workspace or on the instance), and not an active instance superadmin either, fails the + pull rather than falling back to someone else. A superadmin who is not a member runs it + under their instance username, and only while no member of the workspace holds that + username: `u/` resolves through the workspace's members before the email. A + repository whose settings predate the stamp runs as the workspace's first active admin + until they are saved again. +- The identity is resolved before the deploy check is posted, and a failure to resolve it + or to enqueue is recorded on the repository's status, not returned: a returned error + would fail the webhook delivery, and hosts disable hooks whose deliveries keep failing. + The next push or poll retries. +- Fork pulls run as the parent repository's identity, stamped or not, resolved in the + parent (revoking that admin there stops fork pulls too), and first add that admin to + the fork as an admin member, since a plain fork carries only its creator. The fork's + owner cannot be the identity: a non-admin's `wmill sync push` diffs against what it can + see, so an item in a folder it cannot read reads as a create and the push fails on every + commit. CI tests do run as the owner (Phase 7), because they only execute. +- Known and accepted: repo writers control the pull's includes through `wmill.yaml`, so a + fork's owner can commit a user file that makes them admin of the fork and read the + parent secrets it cloned, as with the `push-on-merge-to-forks` Action this replaces. + Loop prevention (pull → deploys → deployment callback → commit → push event): 1. Skip events whose sender is the app bot (`windmill-sync-helper[bot]` / diff --git a/frontend/src/lib/components/git_sync/GitSyncContext.svelte.ts b/frontend/src/lib/components/git_sync/GitSyncContext.svelte.ts index 0150e329ce..075c6b1f24 100644 --- a/frontend/src/lib/components/git_sync/GitSyncContext.svelte.ts +++ b/frontend/src/lib/components/git_sync/GitSyncContext.svelte.ts @@ -1,5 +1,5 @@ import { getContext, setContext } from 'svelte' -import { enterpriseLicense } from '$lib/stores' +import { enterpriseLicense, userStore } from '$lib/stores' import { get } from 'svelte/store' import { sendUserToast } from '$lib/toast' import { apiErrorMessage } from '$lib/utils' @@ -534,6 +534,15 @@ export function createGitSyncContext(workspace: string) { } }) + // The server stamps the saving admin as who pulls run as; mirror it so the card + // names them without a reload. + if (repoToSave.auto_pull) { + repoToSave.auto_pull = { + ...repoToSave.auto_pull, + enabled_by: repoToSave.auto_pull.enabled ? get(userStore)?.email : undefined + } + } + // Update local state with migrated repository repositories[idx] = repoToSave initialRepositories[idx] = { ...repoToSave } diff --git a/frontend/src/lib/components/git_sync/GitSyncRepositoryCard.svelte b/frontend/src/lib/components/git_sync/GitSyncRepositoryCard.svelte index 4b4bff2168..32a510c54a 100644 --- a/frontend/src/lib/components/git_sync/GitSyncRepositoryCard.svelte +++ b/frontend/src/lib/components/git_sync/GitSyncRepositoryCard.svelte @@ -1038,6 +1038,14 @@ : 'Checking the tracked branch about every minute. New commits deploy automatically.'} {/if}
    + {#if repo.auto_pull?.enabled_by} +
    + Pulls apply changes as {repo.auto_pull.enabled_by}, the admin who last saved + these settings{repo.auto_pull.sync_forks + ? ', in this workspace and its forks' + : ''}. +
    + {/if} {#if hasManagedCredential && repo.auto_pull?.webhook_error}
    From 42f489685bc87a8479818175c87b5a21b0c17998 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Tue, 15 Sep 2026 11:32:50 +0200 Subject: [PATCH 123/144] feat: store resource type display names and label hub integrations (#11113) * feat: label resource types and integrations with hub display names Co-Authored-By: Claude Opus 5 (1M context) * fix: load hub integration names in the app and flow pickers Co-Authored-By: Claude Opus 5 (1M context) * fix: load hub resource type names where drawers title a type Co-Authored-By: Claude Opus 5 (1M context) * feat: store resource type display names and drop the hardcoded list Co-Authored-By: Claude Opus 5 (1M context) * fix: leave display_name out of the fork comparison Co-Authored-By: Claude Opus 5 (1M context) * fix: ignore over-long synced display names, move name loaders Co-Authored-By: Claude Opus 5 (1M context) * fix: share the hub integration list cache, backfill admins only Co-Authored-By: Claude Opus 5 (1M context) * fix: keep a name over a nameless duplicate, retry failed hub reads Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Opus 5 (1M context) --- ...04befc3a6596c258086805dd42f513ddb3ead.json | 20 ++++ ...6b07cdb70994be5724f08f864d29517f4907.json} | 12 +- ...c5dd6c7b0da4f76131313d3e6d97f36c5c34.json} | 6 +- ...ecdad014ce319066eabec0f0d7f53338698a.json} | 12 +- ...f4d070f803b6192739e9ebaad7427ce59251.json} | 12 +- ...e795ff9fb6b863dfe9c0bcad382c3b73a25b.json} | 12 +- ...76488dc6c80428e13e5db3c016e37072cfe4.json} | 4 +- ...2e2e8bc43633707d8365f73130c5bca3923b9.json | 18 --- ...95bd6b3fb6ebf3ba115160573487cee4a606.json} | 7 +- ...131148_resource_type_display_name.down.sql | 1 + ...14131148_resource_type_display_name.up.sql | 24 ++++ backend/src/main.rs | 67 +++++++++-- backend/summarized_schema.txt | 2 +- backend/windmill-api-embeddings/src/lib.rs | 2 +- .../tests/resources.rs | 24 ++++ backend/windmill-api-settings/src/lib.rs | 30 ++++- .../windmill-api-workspaces/src/workspaces.rs | 4 +- backend/windmill-api/openapi.yaml | 15 +++ backend/windmill-api/src/workspaces_export.rs | 2 +- backend/windmill-store/src/resources.rs | 42 ++++++- cli/src/commands/hub/hub.ts | 7 +- .../commands/resource-type/resource-type.ts | 1 + .../src/lib/components/AppConnectInner.svelte | 4 + .../src/lib/components/ImportSetupStep.svelte | 19 +++- .../LightweightResourcePicker.svelte | 2 + .../components/ResourceEditorDrawer.svelte | 3 + .../lib/components/ResourceTypePicker.svelte | 7 +- .../src/lib/components/displayNameLoaders.ts | 71 ++++++++++++ .../flows/pickers/PickHubApp.svelte | 2 + .../flows/pickers/PickHubFlow.svelte | 2 + .../flows/pickers/PickHubScript.svelte | 5 +- .../flows/pickers/PickHubScriptQuick.svelte | 12 +- .../lib/components/home/ListFilters.svelte | 3 +- .../components/home/ListFiltersQuick.svelte | 3 +- .../components/mcp/McpScopeSelector.svelte | 15 +-- .../src/lib/components/resourceTypeDisplay.ts | 105 +++++++++++++----- .../components/resourceTypeMatchRank.test.ts | 46 +++++++- .../(root)/(logged)/resources/+page.svelte | 15 +-- 38 files changed, 522 insertions(+), 116 deletions(-) create mode 100644 backend/.sqlx/query-212bf5b32de102a9c537907aaff04befc3a6596c258086805dd42f513ddb3ead.json rename backend/.sqlx/{query-623b061ccaa6bb883e95771fde8c911a165c9c430b7db389370361ca74d737f4.json => query-36ddecbdad3cce7a2593171ff10a6b07cdb70994be5724f08f864d29517f4907.json} (76%) rename backend/.sqlx/{query-8ad79b80033b38ebddf6c8cd4d8cb160d41bac4c45a0fc74d9c9e96d3ef4486a.json => query-386e14cf7572027f2c4ef313cd7cc5dd6c7b0da4f76131313d3e6d97f36c5c34.json} (70%) rename backend/.sqlx/{query-d0a95698b9a2c5e2543e94276d854d7e509c7db2c2ac7d395b7b53ad5dbc25e6.json => query-38b6c6cb91d3ba38838a7a015c59ecdad014ce319066eabec0f0d7f53338698a.json} (75%) rename backend/.sqlx/{query-e253b9e7e6450652589d6ee7ffa86d600e449cd399ac781af8b40c1c444972c3.json => query-6f993567336a2f5ff642ed54e3aaf4d070f803b6192739e9ebaad7427ce59251.json} (78%) rename backend/.sqlx/{query-45d5e9ead8193a04fd00c44a488590fdd2f7c4de45117a18360651655d153545.json => query-82350027cf9722a993f27808e570e795ff9fb6b863dfe9c0bcad382c3b73a25b.json} (78%) rename backend/.sqlx/{query-1c2157ce14e90f0751d7f0a9f2dbb3c5a5789a32423e75260098a5300a4af986.json => query-86ad1e7ebe659f97877cc142c09676488dc6c80428e13e5db3c016e37072cfe4.json} (52%) delete mode 100644 backend/.sqlx/query-972df41db505fbbd20a558b200a2e2e8bc43633707d8365f73130c5bca3923b9.json rename backend/.sqlx/{query-5899c7614f195fdd23e38389e52b004f957aafa2201b80638b5f87a625373f00.json => query-dd6f4b505f4c1e2c734c5d04528c95bd6b3fb6ebf3ba115160573487cee4a606.json} (64%) create mode 100644 backend/migrations/20260914131148_resource_type_display_name.down.sql create mode 100644 backend/migrations/20260914131148_resource_type_display_name.up.sql create mode 100644 frontend/src/lib/components/displayNameLoaders.ts diff --git a/backend/.sqlx/query-212bf5b32de102a9c537907aaff04befc3a6596c258086805dd42f513ddb3ead.json b/backend/.sqlx/query-212bf5b32de102a9c537907aaff04befc3a6596c258086805dd42f513ddb3ead.json new file mode 100644 index 0000000000..04b23832d9 --- /dev/null +++ b/backend/.sqlx/query-212bf5b32de102a9c537907aaff04befc3a6596c258086805dd42f513ddb3ead.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO resource_type (workspace_id, name, schema, description, format_extension, display_name, edited_at)\n VALUES ('admins', $1, $2, $3, $4, $6, now())\n ON CONFLICT (workspace_id, name) DO UPDATE\n SET schema = EXCLUDED.schema, description = EXCLUDED.description,\n -- A fileset is a set of files, so it cannot also be one file.\n -- Create and update reject the pair; this writer bypasses both, so\n -- it declines the extension rather than persisting the forbidden\n -- combination onto a same-named local fileset.\n format_extension = CASE\n WHEN resource_type.is_fileset THEN NULL\n WHEN $5 THEN EXCLUDED.format_extension\n ELSE resource_type.format_extension END,\n display_name = CASE WHEN $7 THEN EXCLUDED.display_name ELSE resource_type.display_name END,\n edited_at = now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Jsonb", + "Text", + "Varchar", + "Bool", + "Varchar", + "Bool" + ] + }, + "nullable": [] + }, + "hash": "212bf5b32de102a9c537907aaff04befc3a6596c258086805dd42f513ddb3ead" +} diff --git a/backend/.sqlx/query-623b061ccaa6bb883e95771fde8c911a165c9c430b7db389370361ca74d737f4.json b/backend/.sqlx/query-36ddecbdad3cce7a2593171ff10a6b07cdb70994be5724f08f864d29517f4907.json similarity index 76% rename from backend/.sqlx/query-623b061ccaa6bb883e95771fde8c911a165c9c430b7db389370361ca74d737f4.json rename to backend/.sqlx/query-36ddecbdad3cce7a2593171ff10a6b07cdb70994be5724f08f864d29517f4907.json index d2084d76d8..76e897d3a4 100644 --- a/backend/.sqlx/query-623b061ccaa6bb883e95771fde8c911a165c9c430b7db389370361ca74d737f4.json +++ b/backend/.sqlx/query-36ddecbdad3cce7a2593171ff10a6b07cdb70994be5724f08f864d29517f4907.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset from resource_type WHERE name = $1 AND (workspace_id = $2 OR workspace_id = 'admins')", + "query": "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset, display_name from resource_type WHERE name = $1 AND (workspace_id = $2 OR workspace_id = 'admins')", "describe": { "columns": [ { @@ -42,6 +42,11 @@ "ordinal": 7, "name": "is_fileset", "type_info": "Bool" + }, + { + "ordinal": 8, + "name": "display_name", + "type_info": "Varchar" } ], "parameters": { @@ -58,8 +63,9 @@ true, true, true, - false + false, + true ] }, - "hash": "623b061ccaa6bb883e95771fde8c911a165c9c430b7db389370361ca74d737f4" + "hash": "36ddecbdad3cce7a2593171ff10a6b07cdb70994be5724f08f864d29517f4907" } diff --git a/backend/.sqlx/query-8ad79b80033b38ebddf6c8cd4d8cb160d41bac4c45a0fc74d9c9e96d3ef4486a.json b/backend/.sqlx/query-386e14cf7572027f2c4ef313cd7cc5dd6c7b0da4f76131313d3e6d97f36c5c34.json similarity index 70% rename from backend/.sqlx/query-8ad79b80033b38ebddf6c8cd4d8cb160d41bac4c45a0fc74d9c9e96d3ef4486a.json rename to backend/.sqlx/query-386e14cf7572027f2c4ef313cd7cc5dd6c7b0da4f76131313d3e6d97f36c5c34.json index 3b9a2e2f34..2f794e6be0 100644 --- a/backend/.sqlx/query-8ad79b80033b38ebddf6c8cd4d8cb160d41bac4c45a0fc74d9c9e96d3ef4486a.json +++ b/backend/.sqlx/query-386e14cf7572027f2c4ef313cd7cc5dd6c7b0da4f76131313d3e6d97f36c5c34.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT EXISTS(SELECT 1 FROM resource_type WHERE workspace_id = 'admins' AND name = $1 AND schema IS NOT DISTINCT FROM $2 AND description IS NOT DISTINCT FROM $3 AND ($5 IS NOT TRUE OR format_extension IS NOT DISTINCT FROM $4))", + "query": "SELECT EXISTS(SELECT 1 FROM resource_type WHERE workspace_id = 'admins' AND name = $1 AND schema IS NOT DISTINCT FROM $2 AND description IS NOT DISTINCT FROM $3 AND ($5 IS NOT TRUE OR format_extension IS NOT DISTINCT FROM $4) AND ($7 IS NOT TRUE OR display_name IS NOT DISTINCT FROM $6))", "describe": { "columns": [ { @@ -15,6 +15,8 @@ "Jsonb", "Text", "Text", + "Bool", + "Text", "Bool" ] }, @@ -22,5 +24,5 @@ null ] }, - "hash": "8ad79b80033b38ebddf6c8cd4d8cb160d41bac4c45a0fc74d9c9e96d3ef4486a" + "hash": "386e14cf7572027f2c4ef313cd7cc5dd6c7b0da4f76131313d3e6d97f36c5c34" } diff --git a/backend/.sqlx/query-d0a95698b9a2c5e2543e94276d854d7e509c7db2c2ac7d395b7b53ad5dbc25e6.json b/backend/.sqlx/query-38b6c6cb91d3ba38838a7a015c59ecdad014ce319066eabec0f0d7f53338698a.json similarity index 75% rename from backend/.sqlx/query-d0a95698b9a2c5e2543e94276d854d7e509c7db2c2ac7d395b7b53ad5dbc25e6.json rename to backend/.sqlx/query-38b6c6cb91d3ba38838a7a015c59ecdad014ce319066eabec0f0d7f53338698a.json index ffd1670c04..87d740a247 100644 --- a/backend/.sqlx/query-d0a95698b9a2c5e2543e94276d854d7e509c7db2c2ac7d395b7b53ad5dbc25e6.json +++ b/backend/.sqlx/query-38b6c6cb91d3ba38838a7a015c59ecdad014ce319066eabec0f0d7f53338698a.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset from resource_type WHERE (workspace_id = $1 OR workspace_id = 'admins') ORDER BY name", + "query": "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset, display_name from resource_type WHERE (workspace_id = $1 OR workspace_id = 'admins') ORDER BY name", "describe": { "columns": [ { @@ -42,6 +42,11 @@ "ordinal": 7, "name": "is_fileset", "type_info": "Bool" + }, + { + "ordinal": 8, + "name": "display_name", + "type_info": "Varchar" } ], "parameters": { @@ -57,8 +62,9 @@ true, true, true, - false + false, + true ] }, - "hash": "d0a95698b9a2c5e2543e94276d854d7e509c7db2c2ac7d395b7b53ad5dbc25e6" + "hash": "38b6c6cb91d3ba38838a7a015c59ecdad014ce319066eabec0f0d7f53338698a" } diff --git a/backend/.sqlx/query-e253b9e7e6450652589d6ee7ffa86d600e449cd399ac781af8b40c1c444972c3.json b/backend/.sqlx/query-6f993567336a2f5ff642ed54e3aaf4d070f803b6192739e9ebaad7427ce59251.json similarity index 78% rename from backend/.sqlx/query-e253b9e7e6450652589d6ee7ffa86d600e449cd399ac781af8b40c1c444972c3.json rename to backend/.sqlx/query-6f993567336a2f5ff642ed54e3aaf4d070f803b6192739e9ebaad7427ce59251.json index c44d3d711d..e444c0d549 100644 --- a/backend/.sqlx/query-e253b9e7e6450652589d6ee7ffa86d600e449cd399ac781af8b40c1c444972c3.json +++ b/backend/.sqlx/query-6f993567336a2f5ff642ed54e3aaf4d070f803b6192739e9ebaad7427ce59251.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset from resource_type ORDER BY name", + "query": "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset, display_name from resource_type ORDER BY name", "describe": { "columns": [ { @@ -42,6 +42,11 @@ "ordinal": 7, "name": "is_fileset", "type_info": "Bool" + }, + { + "ordinal": 8, + "name": "display_name", + "type_info": "Varchar" } ], "parameters": { @@ -55,8 +60,9 @@ true, true, true, - false + false, + true ] }, - "hash": "e253b9e7e6450652589d6ee7ffa86d600e449cd399ac781af8b40c1c444972c3" + "hash": "6f993567336a2f5ff642ed54e3aaf4d070f803b6192739e9ebaad7427ce59251" } diff --git a/backend/.sqlx/query-45d5e9ead8193a04fd00c44a488590fdd2f7c4de45117a18360651655d153545.json b/backend/.sqlx/query-82350027cf9722a993f27808e570e795ff9fb6b863dfe9c0bcad382c3b73a25b.json similarity index 78% rename from backend/.sqlx/query-45d5e9ead8193a04fd00c44a488590fdd2f7c4de45117a18360651655d153545.json rename to backend/.sqlx/query-82350027cf9722a993f27808e570e795ff9fb6b863dfe9c0bcad382c3b73a25b.json index e4db87ec7d..9f7be9d7cb 100644 --- a/backend/.sqlx/query-45d5e9ead8193a04fd00c44a488590fdd2f7c4de45117a18360651655d153545.json +++ b/backend/.sqlx/query-82350027cf9722a993f27808e570e795ff9fb6b863dfe9c0bcad382c3b73a25b.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset FROM resource_type WHERE workspace_id = $1", + "query": "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset, display_name FROM resource_type WHERE workspace_id = $1", "describe": { "columns": [ { @@ -42,6 +42,11 @@ "ordinal": 7, "name": "is_fileset", "type_info": "Bool" + }, + { + "ordinal": 8, + "name": "display_name", + "type_info": "Varchar" } ], "parameters": { @@ -57,8 +62,9 @@ true, true, true, - false + false, + true ] }, - "hash": "45d5e9ead8193a04fd00c44a488590fdd2f7c4de45117a18360651655d153545" + "hash": "82350027cf9722a993f27808e570e795ff9fb6b863dfe9c0bcad382c3b73a25b" } diff --git a/backend/.sqlx/query-1c2157ce14e90f0751d7f0a9f2dbb3c5a5789a32423e75260098a5300a4af986.json b/backend/.sqlx/query-86ad1e7ebe659f97877cc142c09676488dc6c80428e13e5db3c016e37072cfe4.json similarity index 52% rename from backend/.sqlx/query-1c2157ce14e90f0751d7f0a9f2dbb3c5a5789a32423e75260098a5300a4af986.json rename to backend/.sqlx/query-86ad1e7ebe659f97877cc142c09676488dc6c80428e13e5db3c016e37072cfe4.json index fc354fb9a2..f417ab94f9 100644 --- a/backend/.sqlx/query-1c2157ce14e90f0751d7f0a9f2dbb3c5a5789a32423e75260098a5300a4af986.json +++ b/backend/.sqlx/query-86ad1e7ebe659f97877cc142c09676488dc6c80428e13e5db3c016e37072cfe4.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO resource_type (workspace_id, name, schema, description, edited_at, created_by, format_extension, is_fileset)\n SELECT $2, name, schema, description, edited_at, created_by, format_extension, is_fileset\n FROM resource_type\n WHERE workspace_id = $1", + "query": "INSERT INTO resource_type (workspace_id, name, schema, description, edited_at, created_by, format_extension, is_fileset, display_name)\n SELECT $2, name, schema, description, edited_at, created_by, format_extension, is_fileset, display_name\n FROM resource_type\n WHERE workspace_id = $1", "describe": { "columns": [], "parameters": { @@ -11,5 +11,5 @@ }, "nullable": [] }, - "hash": "1c2157ce14e90f0751d7f0a9f2dbb3c5a5789a32423e75260098a5300a4af986" + "hash": "86ad1e7ebe659f97877cc142c09676488dc6c80428e13e5db3c016e37072cfe4" } diff --git a/backend/.sqlx/query-972df41db505fbbd20a558b200a2e2e8bc43633707d8365f73130c5bca3923b9.json b/backend/.sqlx/query-972df41db505fbbd20a558b200a2e2e8bc43633707d8365f73130c5bca3923b9.json deleted file mode 100644 index 3624a32893..0000000000 --- a/backend/.sqlx/query-972df41db505fbbd20a558b200a2e2e8bc43633707d8365f73130c5bca3923b9.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO resource_type (workspace_id, name, schema, description, format_extension, edited_at)\n VALUES ('admins', $1, $2, $3, $4, now())\n ON CONFLICT (workspace_id, name) DO UPDATE\n SET schema = EXCLUDED.schema, description = EXCLUDED.description,\n -- A fileset is a set of files, so it cannot also be one file.\n -- Create and update reject the pair; this writer bypasses both, so\n -- it declines the extension rather than persisting the forbidden\n -- combination onto a same-named local fileset.\n format_extension = CASE\n WHEN resource_type.is_fileset THEN NULL\n WHEN $5 THEN EXCLUDED.format_extension\n ELSE resource_type.format_extension END,\n edited_at = now()", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Jsonb", - "Text", - "Varchar", - "Bool" - ] - }, - "nullable": [] - }, - "hash": "972df41db505fbbd20a558b200a2e2e8bc43633707d8365f73130c5bca3923b9" -} diff --git a/backend/.sqlx/query-5899c7614f195fdd23e38389e52b004f957aafa2201b80638b5f87a625373f00.json b/backend/.sqlx/query-dd6f4b505f4c1e2c734c5d04528c95bd6b3fb6ebf3ba115160573487cee4a606.json similarity index 64% rename from backend/.sqlx/query-5899c7614f195fdd23e38389e52b004f957aafa2201b80638b5f87a625373f00.json rename to backend/.sqlx/query-dd6f4b505f4c1e2c734c5d04528c95bd6b3fb6ebf3ba115160573487cee4a606.json index 400c8d9ee6..340b09ca27 100644 --- a/backend/.sqlx/query-5899c7614f195fdd23e38389e52b004f957aafa2201b80638b5f87a625373f00.json +++ b/backend/.sqlx/query-dd6f4b505f4c1e2c734c5d04528c95bd6b3fb6ebf3ba115160573487cee4a606.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO resource_type\n (workspace_id, name, schema, description, created_by, format_extension, is_fileset, edited_at)\n VALUES ($1, $2, $3, $4, $5, $6, $7, now())", + "query": "INSERT INTO resource_type\n (workspace_id, name, schema, description, created_by, format_extension, is_fileset, display_name, edited_at)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, now())", "describe": { "columns": [], "parameters": { @@ -11,10 +11,11 @@ "Text", "Varchar", "Varchar", - "Bool" + "Bool", + "Varchar" ] }, "nullable": [] }, - "hash": "5899c7614f195fdd23e38389e52b004f957aafa2201b80638b5f87a625373f00" + "hash": "dd6f4b505f4c1e2c734c5d04528c95bd6b3fb6ebf3ba115160573487cee4a606" } diff --git a/backend/migrations/20260914131148_resource_type_display_name.down.sql b/backend/migrations/20260914131148_resource_type_display_name.down.sql new file mode 100644 index 0000000000..57c9ffabc8 --- /dev/null +++ b/backend/migrations/20260914131148_resource_type_display_name.down.sql @@ -0,0 +1 @@ +ALTER TABLE resource_type DROP COLUMN display_name; diff --git a/backend/migrations/20260914131148_resource_type_display_name.up.sql b/backend/migrations/20260914131148_resource_type_display_name.up.sql new file mode 100644 index 0000000000..213622ef94 --- /dev/null +++ b/backend/migrations/20260914131148_resource_type_display_name.up.sql @@ -0,0 +1,24 @@ +-- The name a product goes by, beside the identifier a resource references: `gsheets` is +-- "Google Sheets". Null where nobody named the type; readers derive a label from the name. +ALTER TABLE resource_type ADD COLUMN display_name VARCHAR(100); + +-- The names the hub carries today, so existing instances show them before any sync. Only in +-- admins, where hub resource types live and every workspace reads them from. +UPDATE resource_type SET display_name = v.display_name +FROM (VALUES + ('bamboo_hr', 'BambooHR'), + ('cacertificate', 'CA certificate'), + ('deep_infra', 'DeepInfra'), + ('gcal', 'Google Calendar'), + ('gdocs', 'Google Docs'), + ('gdrive', 'Google Drive'), + ('gforms', 'Google Forms'), + ('gsheets', 'Google Sheets'), + ('gworkspace', 'Google Workspace'), + ('sensortower', 'Sensor Tower'), + ('snowflake_oauth', 'Snowflake (OAuth)'), + ('their_stack', 'TheirStack') +) AS v(name, display_name) +WHERE resource_type.workspace_id = 'admins' + AND resource_type.name = v.name + AND resource_type.display_name IS NULL; diff --git a/backend/src/main.rs b/backend/src/main.rs index 7f2ce3cd82..316ed099f2 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -411,6 +411,13 @@ struct HubResourceTypeRaw { /// Absent from hubs predating the column, and from caches written before it. #[serde(default)] pub format_extension: Option, + /// Doubly optional, so a hub predating the field (no key) is told apart from a type the + /// hub leaves unnamed (null). + #[serde( + default, + deserialize_with = "windmill_common::more_serde::double_option" + )] + pub display_name: Option>, } @@ -434,6 +441,14 @@ pub struct HubResourceType { skip_serializing_if = "Option::is_none" )] pub format_extension: Option>, + /// Doubly optional like `format_extension`: a cache written before the field leaves the + /// stored name alone, while a null from the hub clears it. + #[serde( + default, + deserialize_with = "windmill_common::more_serde::double_option", + skip_serializing_if = "Option::is_none" + )] + pub display_name: Option>, } const HUB_RT_CACHE_FILE: &str = "resource_types.json"; @@ -481,6 +496,7 @@ async fn cache_hub_resource_types() -> anyhow::Result<()> { app: rt.app, description: rt.description, format_extension: Some(rt.format_extension), + display_name: rt.display_name, }) }) .collect(); @@ -531,8 +547,9 @@ pub async fn sync_cached_resource_types(db: &sqlx::Pool) -> anyh Option, Option, bool, + Option, )> = sqlx::query_as( - "SELECT name, schema, description, format_extension, is_fileset FROM resource_type WHERE workspace_id = 'admins'", + "SELECT name, schema, description, format_extension, is_fileset, display_name FROM resource_type WHERE workspace_id = 'admins'", ) .fetch_all(db) .await @@ -540,12 +557,23 @@ pub async fn sync_cached_resource_types(db: &sqlx::Pool) -> anyh let existing_map: std::collections::HashMap< String, - (Option, Option, Option, bool), + ( + Option, + Option, + Option, + bool, + Option, + ), > = existing_types .into_iter() - .map(|(name, schema, desc, format_extension, is_fileset)| { - (name, (schema, desc, format_extension, is_fileset)) - }) + .map( + |(name, schema, desc, format_extension, is_fileset, display_name)| { + ( + name, + (schema, desc, format_extension, is_fileset, display_name), + ) + }, + ) .collect(); let mut synced_count = 0; @@ -553,8 +581,9 @@ pub async fn sync_cached_resource_types(db: &sqlx::Pool) -> anyh for rt in cached_types { let existing = existing_map.get(&rt.name); - let is_fileset = existing.map(|(_, _, _, f)| *f).unwrap_or(false); - let stored_extension = existing.and_then(|(_, _, e, _)| e.clone()); + let is_fileset = existing.map(|(_, _, _, f, _)| *f).unwrap_or(false); + let stored_extension = existing.and_then(|(_, _, e, _, _)| e.clone()); + let stored_display_name = existing.and_then(|(_, _, _, _, n)| n.clone()); // A fileset is a set of files, so it cannot also be one file. Create, update // and the manual sync all reject the pair; this writer would otherwise // persist it onto a same-named local fileset. @@ -570,11 +599,25 @@ pub async fn sync_cached_resource_types(db: &sqlx::Pool) -> anyh None => stored_extension.clone(), } }; + // No key in the cache leaves the stored name alone, as for the extension. So does a name + // too long for the column: one bad entry must not fail the upsert and end the sync. + let display_name = match &rt.display_name { + Some(Some(name)) if name.chars().count() > 100 => { + tracing::warn!( + "Ignoring the display_name of resource type {}: longer than 100 characters", + rt.name + ); + stored_display_name.clone() + } + Some(from_cache) => from_cache.clone(), + None => stored_display_name.clone(), + }; - if let Some((existing_schema, existing_desc, _, _)) = existing { + if let Some((existing_schema, existing_desc, _, _, _)) = existing { if existing_schema == &rt.schema && existing_desc == &rt.description && stored_extension == format_extension + && stored_display_name == display_name { skipped_count += 1; continue; @@ -586,16 +629,18 @@ pub async fn sync_cached_resource_types(db: &sqlx::Pool) -> anyh // `format_extension` is resolved above rather than coalesced here: a // COALESCE could never clear one, so a hub that dropped an extension // would leave the stale value behind forever. - "INSERT INTO resource_type (workspace_id, name, schema, description, format_extension, edited_at) - VALUES ('admins', $1, $2, $3, $4, now()) + "INSERT INTO resource_type (workspace_id, name, schema, description, format_extension, display_name, edited_at) + VALUES ('admins', $1, $2, $3, $4, $5, now()) ON CONFLICT (workspace_id, name) DO UPDATE SET schema = EXCLUDED.schema, description = EXCLUDED.description, - format_extension = EXCLUDED.format_extension, edited_at = now()", + format_extension = EXCLUDED.format_extension, + display_name = EXCLUDED.display_name, edited_at = now()", ) .bind(&rt.name) .bind(&rt.schema) .bind(&rt.description) .bind(&format_extension) + .bind(&display_name) .execute(db) .await .with_context(|| format!("Failed to upsert resource type {}", rt.name))?; diff --git a/backend/summarized_schema.txt b/backend/summarized_schema.txt index 1ab05bc584..463afd0d33 100644 --- a/backend/summarized_schema.txt +++ b/backend/summarized_schema.txt @@ -173,7 +173,7 @@ raw_app: path(char), version(int), workspace_id(char), summary(char), edited_at( FK: (workspace_id) -> workspace(id) resource: workspace_id(char), path(char), value(jsonb), description(text), resource_type(char), extra_perms(jsonb), edited_at(ts), created_by(char), labels(text[]) FK: (workspace_id) -> workspace(id) -resource_type: workspace_id(char), name(char), schema(jsonb), description(text), edited_at(ts), created_by(char), format_extension(char), is_fileset(bool) +resource_type: workspace_id(char), name(char), schema(jsonb), description(text), edited_at(ts), created_by(char), format_extension(char), is_fileset(bool), display_name(char) FK: (workspace_id) -> workspace(id) resume_job: id(uuid), job(uuid), flow(uuid), created_at(ts), value(jsonb), approver(char), resume_id(int), approved(bool) FK: (flow) -> v2_job_queue(id) diff --git a/backend/windmill-api-embeddings/src/lib.rs b/backend/windmill-api-embeddings/src/lib.rs index 35816dfaa8..1de99fb44f 100644 --- a/backend/windmill-api-embeddings/src/lib.rs +++ b/backend/windmill-api-embeddings/src/lib.rs @@ -418,7 +418,7 @@ impl EmbeddingsDb { let hub_resource_types = response.json::>().await?; let resource_types: Vec = - sqlx::query_as!(ResourceType, "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset from resource_type ORDER BY name",) + sqlx::query_as!(ResourceType, "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset, display_name from resource_type ORDER BY name",) .fetch_all(pg_db) .await?; diff --git a/backend/windmill-api-integration-tests/tests/resources.rs b/backend/windmill-api-integration-tests/tests/resources.rs index 3f0b21b184..53648ec759 100644 --- a/backend/windmill-api-integration-tests/tests/resources.rs +++ b/backend/windmill-api-integration-tests/tests/resources.rs @@ -443,6 +443,30 @@ async fn test_resource_endpoints(db: Pool) -> anyhow::Result<()> { let body = resp.json::().await?; assert_eq!(body["description"], "Updated type desc"); + // display_name: an update that omits it, as a push from a CLI predating the field does, + // keeps it; an explicit null clears it. + for (update, expected) in [ + ( + json!({"display_name": "New Test Type"}), + json!("New Test Type"), + ), + ( + json!({"description": "Updated type desc"}), + json!("New Test Type"), + ), + (json!({"display_name": null}), serde_json::Value::Null), + ] { + let resp = authed(client().post(resource_url(port, "type/update", "new_test_type"))) + .json(&update) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let resp = authed_get(port, "type/get", "new_test_type").await; + let body = resp.json::().await?; + assert_eq!(body["display_name"], expected); + } + // type/delete let resp = authed(client().delete(resource_url(port, "type/delete", "new_test_type"))) .send() diff --git a/backend/windmill-api-settings/src/lib.rs b/backend/windmill-api-settings/src/lib.rs index 0fe358da1c..2e2b3290fa 100644 --- a/backend/windmill-api-settings/src/lib.rs +++ b/backend/windmill-api-settings/src/lib.rs @@ -2074,6 +2074,13 @@ struct CachedResourceType { deserialize_with = "windmill_common::more_serde::double_option" )] format_extension: Option>, + /// Doubly optional like `format_extension`: no key leaves the stored name alone, an explicit + /// null (the hub naming nothing) clears it. + #[serde( + default, + deserialize_with = "windmill_common::more_serde::double_option" + )] + display_name: Option>, } #[derive(serde::Deserialize)] @@ -2085,6 +2092,11 @@ struct HubResourceTypeRaw { description: Option, #[serde(default)] format_extension: Option, + #[serde( + default, + deserialize_with = "windmill_common::more_serde::double_option" + )] + display_name: Option>, } async fn fetch_resource_types_from_hub() -> error::Result> { @@ -2127,6 +2139,7 @@ async fn fetch_resource_types_from_hub() -> error::Result 100 => None, + other => other.clone(), + }; let exists: Option = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM resource_type WHERE workspace_id = 'admins' AND name = $1 AND schema IS NOT DISTINCT FROM $2 AND description IS NOT DISTINCT FROM $3 AND ($5 IS NOT TRUE OR format_extension IS NOT DISTINCT FROM $4))", + "SELECT EXISTS(SELECT 1 FROM resource_type WHERE workspace_id = 'admins' AND name = $1 AND schema IS NOT DISTINCT FROM $2 AND description IS NOT DISTINCT FROM $3 AND ($5 IS NOT TRUE OR format_extension IS NOT DISTINCT FROM $4) AND ($7 IS NOT TRUE OR display_name IS NOT DISTINCT FROM $6))", &rt.name, rt.schema.as_ref(), rt.description.as_deref(), rt.format_extension.clone().flatten(), rt.format_extension.is_some(), + display_name.clone().flatten(), + display_name.is_some(), ) .fetch_one(&db) .await?; @@ -2198,8 +2219,8 @@ async fn sync_cached_resource_types( // Whether the payload carried the key at all is what decides: present // (even as null) is authoritative and may clear, absent means a cache // written before the column and must leave the stored value alone. - "INSERT INTO resource_type (workspace_id, name, schema, description, format_extension, edited_at) - VALUES ('admins', $1, $2, $3, $4, now()) + "INSERT INTO resource_type (workspace_id, name, schema, description, format_extension, display_name, edited_at) + VALUES ('admins', $1, $2, $3, $4, $6, now()) ON CONFLICT (workspace_id, name) DO UPDATE SET schema = EXCLUDED.schema, description = EXCLUDED.description, -- A fileset is a set of files, so it cannot also be one file. @@ -2210,12 +2231,15 @@ async fn sync_cached_resource_types( WHEN resource_type.is_fileset THEN NULL WHEN $5 THEN EXCLUDED.format_extension ELSE resource_type.format_extension END, + display_name = CASE WHEN $7 THEN EXCLUDED.display_name ELSE resource_type.display_name END, edited_at = now()", &rt.name, rt.schema.as_ref(), rt.description.as_deref(), rt.format_extension.clone().flatten(), rt.format_extension.is_some(), + display_name.clone().flatten(), + display_name.is_some(), ) .execute(&db) .await?; diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 640c0fe06a..6048772ff6 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -6491,8 +6491,8 @@ async fn clone_resource_types( target_workspace_id: &str, ) -> Result<()> { sqlx::query!( - "INSERT INTO resource_type (workspace_id, name, schema, description, edited_at, created_by, format_extension, is_fileset) - SELECT $2, name, schema, description, edited_at, created_by, format_extension, is_fileset + "INSERT INTO resource_type (workspace_id, name, schema, description, edited_at, created_by, format_extension, is_fileset, display_name) + SELECT $2, name, schema, description, edited_at, created_by, format_extension, is_fileset, display_name FROM resource_type WHERE workspace_id = $1", source_workspace_id, diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index b733157d85..21565ffaf7 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -9280,6 +9280,10 @@ paths: picks: description: how often the integration has been picked, absent on a hub that does not count picks type: integer + display_name: + description: the label the hub curates for the integration, null or absent where it names none + type: string + nullable: true required: - name @@ -30777,6 +30781,11 @@ components: type: string is_fileset: type: boolean + display_name: + type: string + description: >- + The name the product goes by, e.g. "Google Sheets" for gsheets. + Absent where nobody named the type. required: - name @@ -30794,6 +30803,12 @@ components: description: >- File extension for a type whose value is one file rather than a set of fields. Omit to leave it unchanged; send null to clear it. + display_name: + type: string + nullable: true + description: >- + The name the product goes by. Omit to leave it unchanged; send null + to clear it. TriggerHistoryEntry: type: object diff --git a/backend/windmill-api/src/workspaces_export.rs b/backend/windmill-api/src/workspaces_export.rs index a85f9d004e..69565ffcbd 100644 --- a/backend/windmill-api/src/workspaces_export.rs +++ b/backend/windmill-api/src/workspaces_export.rs @@ -924,7 +924,7 @@ pub(crate) async fn tarball_workspace( if !skip_resource_types.unwrap_or(false) { let resource_types = sqlx::query_as!( ResourceType, - "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset FROM resource_type WHERE workspace_id = $1", + "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset, display_name FROM resource_type WHERE workspace_id = $1", &w_id ) .fetch_all(&mut *tx) diff --git a/backend/windmill-store/src/resources.rs b/backend/windmill-store/src/resources.rs index 126cf97106..5bb15b456e 100644 --- a/backend/windmill-store/src/resources.rs +++ b/backend/windmill-store/src/resources.rs @@ -118,6 +118,10 @@ pub struct ResourceType { pub edited_at: Option>, pub format_extension: Option, pub is_fileset: bool, + /// The name the product goes by (`gsheets` is "Google Sheets"), null where nobody named it. + /// Skipped when absent, so the type files of a synced repo gain nothing until one is set. + #[serde(skip_serializing_if = "Option::is_none")] + pub display_name: Option, } #[derive(Deserialize)] @@ -127,6 +131,7 @@ pub struct CreateResourceType { pub description: Option, pub format_extension: Option, pub is_fileset: Option, + pub display_name: Option, } #[derive(Deserialize)] @@ -143,6 +148,13 @@ pub struct EditResourceType { deserialize_with = "windmill_common::more_serde::double_option" )] pub format_extension: Option>, + /// Doubly optional for the same reason. A push from a CLI that predates the field omits it, + /// and must not clear a name the hub set. + #[serde( + default, + deserialize_with = "windmill_common::more_serde::double_option" + )] + pub display_name: Option>, } #[derive(FromRow, Serialize, Deserialize)] @@ -2729,7 +2741,7 @@ async fn list_resource_types( ) -> JsonResult> { let rows = sqlx::query_as!( ResourceType, - "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset from resource_type WHERE (workspace_id = $1 OR workspace_id = 'admins') ORDER \ + "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset, display_name from resource_type WHERE (workspace_id = $1 OR workspace_id = 'admins') ORDER \ BY name", &w_id ) @@ -3109,7 +3121,7 @@ async fn get_resource_type( let resource_type_o = sqlx::query_as!( ResourceType, - "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset from resource_type WHERE name = $1 AND (workspace_id = $2 OR workspace_id = 'admins')", + "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset, display_name from resource_type WHERE name = $1 AND (workspace_id = $2 OR workspace_id = 'admins')", &name, &w_id ) @@ -3137,6 +3149,20 @@ async fn exists_resource_type( Ok(Json(exists)) } +/// Trimmed, blank as none, and held to the column's 100 characters, so an over-long name is +/// refused with a message rather than a database error. +fn normalize_display_name(name: Option<&str>) -> Result> { + let Some(name) = name.map(str::trim).filter(|n| !n.is_empty()) else { + return Ok(None); + }; + if name.chars().count() > 100 { + return Err(Error::BadRequest( + "display_name must be at most 100 characters".to_string(), + )); + } + Ok(Some(name.to_string())) +} + async fn create_resource_type( authed: ApiAuthed, Extension(db): Extension, @@ -3170,11 +3196,12 @@ async fn create_resource_type( "A fileset resource type cannot have a format_extension".to_string(), )); } + let display_name = normalize_display_name(resource_type.display_name.as_deref())?; sqlx::query!( "INSERT INTO resource_type - (workspace_id, name, schema, description, created_by, format_extension, is_fileset, edited_at) - VALUES ($1, $2, $3, $4, $5, $6, $7, now())", + (workspace_id, name, schema, description, created_by, format_extension, is_fileset, display_name, edited_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, now())", w_id, resource_type.name, resource_type.schema, @@ -3182,6 +3209,7 @@ async fn create_resource_type( authed.username, resource_type.format_extension, is_fileset, + display_name, ) .execute(&mut *tx) .await?; @@ -3349,6 +3377,12 @@ async fn update_resource_type( None => sqlb.set("format_extension", "NULL"), }; } + if let Some(display_name) = &ns.display_name { + match normalize_display_name(display_name.as_deref())? { + Some(name) => sqlb.set_str("display_name", name), + None => sqlb.set("display_name", "NULL"), + }; + } sqlb.set_str("edited_at", "now()"); let sql = sqlb.sql().map_err(|e| Error::internal_err(e.to_string()))?; let mut tx = user_db.begin(&authed).await?; diff --git a/cli/src/commands/hub/hub.ts b/cli/src/commands/hub/hub.ts index f752147811..1a9c89680e 100644 --- a/cli/src/commands/hub/hub.ts +++ b/cli/src/commands/hub/hub.ts @@ -20,6 +20,9 @@ interface HubResourceType { // Absent from hubs predating the column, so a missing value is "ordinary type", // not "unset it". format_extension?: string | null; + // Null where nobody named the type, and absent from hubs predating the field, which + // leaves a stored name alone rather than clearing it. + display_name?: string | null; } export async function pull(opts: GlobalOptions) { @@ -120,7 +123,9 @@ export async function pull(opts: GlobalOptions) { deepEqual(y.schema, x.schema) && y.description === x.description && (y.is_fileset ?? false) === (x.is_fileset ?? false) && - (y.format_extension ?? null) === (x.format_extension ?? null) + (y.format_extension ?? null) === (x.format_extension ?? null) && + (x.display_name === undefined || + (y.display_name ?? null) === x.display_name) ) ) { log.info("skipping " + x.name + " (same as current)"); diff --git a/cli/src/commands/resource-type/resource-type.ts b/cli/src/commands/resource-type/resource-type.ts index fd4b72108e..505200f07a 100644 --- a/cli/src/commands/resource-type/resource-type.ts +++ b/cli/src/commands/resource-type/resource-type.ts @@ -28,6 +28,7 @@ export interface ResourceTypeFile { // Extension for a type whose value is one file rather than a set of fields; it // is what makes the resource editor a file editor for that language. format_extension?: string | null; + display_name?: string | null; } export async function pushResourceType( diff --git a/frontend/src/lib/components/AppConnectInner.svelte b/frontend/src/lib/components/AppConnectInner.svelte index 99adeac6b3..cef58fcdb2 100644 --- a/frontend/src/lib/components/AppConnectInner.svelte +++ b/frontend/src/lib/components/AppConnectInner.svelte @@ -5,10 +5,12 @@ import LabelsInput from './LabelsInput.svelte' import IconedResourceType from './IconedResourceType.svelte' import { + addResourceTypeDisplayName, isCustomResourceTypeName, resourceTypeDisplayName, resourceTypeMatchRank, resourceTypeSearchText, + setResourceTypeDisplayNames, sortResourceTypesByMatch } from './resourceTypeDisplay' import { @@ -497,6 +499,7 @@ // $derived, so search re-ranks when they land. ResourceService.listResourceType({ workspace: effectiveWorkspace }) .then((types) => { + setResourceTypeDisplayNames(types) resourceTypeDescriptions = Object.fromEntries( types.filter((t) => t.description).map((t) => [t.name, t.description!]) ) @@ -652,6 +655,7 @@ workspace: effectiveWorkspace, path: resourceType }) + addResourceTypeDisplayName(resourceTypeInfo) const props: Record = resourceTypeInfo?.schema?.['properties'] ?? {} const newArgsKeys = Object.keys(props).filter((x) => props?.[x]?.type == 'string') ?? [] diff --git a/frontend/src/lib/components/ImportSetupStep.svelte b/frontend/src/lib/components/ImportSetupStep.svelte index 51eaf725f8..f85ebf7395 100644 --- a/frontend/src/lib/components/ImportSetupStep.svelte +++ b/frontend/src/lib/components/ImportSetupStep.svelte @@ -19,7 +19,11 @@ import { applyRetarget, seesWholeWorkspace } from '$lib/importWizard/retargetDeployed' import { OauthService } from '$lib/gen' import { registryCcCapableFor } from '$lib/components/oauthRegistry' - import { resourceTypeDisplayName } from '$lib/components/resourceTypeDisplay' + import { + addResourceTypeDisplayName, + resourceTypeDisplayName + } from '$lib/components/resourceTypeDisplay' + import { loadResourceTypeDisplayName } from '$lib/components/displayNameLoaders' import { applyOneMigration } from '$lib/components/workspaceSettings/projectInstall' import { probeMigrationsApplied } from '$lib/importWizard/probe' import { @@ -197,6 +201,14 @@ * first half and Connect disappears on the eight such providers, where it would work. */ const canConnectType = (rt: string) => instanceConnects.has(rt) || registryCcCapableFor(rt) + + // A row blocked by a resource of another type names that type, whose row nothing here reads. + $effect(() => { + for (const b of blanks) { + if (b.occupiedBy) void loadResourceTypeDisplayName(workspace, b.occupiedBy) + } + }) + let appConnect: AppConnectDrawer | undefined = $state(undefined) const customInstanceDbs = resource([() => workspace], SettingService.listCustomInstanceDbs) @@ -396,8 +408,9 @@ // row is kept instead; it just cannot name which fields are short. let requirementsUnknown = false try { - const schema = (await ResourceService.getResourceType({ workspace, path: r.resource_type })) - ?.schema as { required?: string[] } | undefined + const rt = await ResourceService.getResourceType({ workspace, path: r.resource_type }) + addResourceTypeDisplayName(rt) + const schema = rt?.schema as { required?: string[] } | undefined required = schema?.required ?? [] } catch { requirementsUnknown = true diff --git a/frontend/src/lib/components/LightweightResourcePicker.svelte b/frontend/src/lib/components/LightweightResourcePicker.svelte index bb0c21b4c1..58557e8c8b 100644 --- a/frontend/src/lib/components/LightweightResourcePicker.svelte +++ b/frontend/src/lib/components/LightweightResourcePicker.svelte @@ -10,6 +10,7 @@ import Select from './select/Select.svelte' import IconedResourceType from './IconedResourceType.svelte' import { addResourceTitle } from './resourceTypeDisplay' + import { loadResourceTypeDisplayName } from './displayNameLoaders' interface Props { value: string | undefined @@ -218,6 +219,7 @@ on:click={() => { refreshCount += 1 open = true + if (ws && resourceType) void loadResourceTypeDisplayName(ws, resourceType) drawer?.openDrawer?.() }} startIcon={{ icon: Plus }} diff --git a/frontend/src/lib/components/ResourceEditorDrawer.svelte b/frontend/src/lib/components/ResourceEditorDrawer.svelte index 36d49d9149..052c8c9e92 100644 --- a/frontend/src/lib/components/ResourceEditorDrawer.svelte +++ b/frontend/src/lib/components/ResourceEditorDrawer.svelte @@ -19,6 +19,7 @@ import ResourceVersionHistory from './ResourceVersionHistory.svelte' import IconedResourceType from './IconedResourceType.svelte' import { addResourceTitle } from './resourceTypeDisplay' + import { loadResourceTypeDisplayName } from './displayNameLoaders' let { workspace = undefined, @@ -111,6 +112,8 @@ // rather than left where the last one put it: a new resource is a typed form, whoever was // looking at JSON before. viewJsonSchema = false + // The title names the type, whose row nothing else on the page may have read. + void loadResourceTypeDisplayName(effectiveWorkspace, resourceType) drawer?.openDrawer?.() } diff --git a/frontend/src/lib/components/ResourceTypePicker.svelte b/frontend/src/lib/components/ResourceTypePicker.svelte index 53dc0ed7a2..35d6593635 100644 --- a/frontend/src/lib/components/ResourceTypePicker.svelte +++ b/frontend/src/lib/components/ResourceTypePicker.svelte @@ -9,7 +9,11 @@ import Tooltip from './Tooltip.svelte' import Badge from './common/badge/Badge.svelte' import { untrack } from 'svelte' - import { resourceTypeSearchText, sortResourceTypesByMatch } from './resourceTypeDisplay' + import { + resourceTypeSearchText, + setResourceTypeDisplayNames, + sortResourceTypesByMatch + } from './resourceTypeDisplay' interface Props { value: string | undefined notPickable?: boolean @@ -22,6 +26,7 @@ async function loadResources() { const types = await ResourceService.listResourceType({ workspace: $workspaceStore! }) + setResourceTypeDisplayNames(types) resources = types.map((t) => ({ name: t.name, description: t.description, diff --git a/frontend/src/lib/components/displayNameLoaders.ts b/frontend/src/lib/components/displayNameLoaders.ts new file mode 100644 index 0000000000..5eed1e81a0 --- /dev/null +++ b/frontend/src/lib/components/displayNameLoaders.ts @@ -0,0 +1,71 @@ +import { get } from 'svelte/store' +import { IntegrationService, ResourceService } from '$lib/gen' +import { disableHubStore } from '$lib/stores' +import { createCache } from '$lib/utils' +import { addResourceTypeDisplayName, setHubIntegrationDisplayNames } from './resourceTypeDisplay' + +/** + * Loads what `resourceTypeDisplayName` and `integrationDisplayName` read: a type's stored name for a + * surface that holds no row for it, and the hub's integration list, which every picker that needs + * it shares. Apart from `resourceTypeDisplay`, which makes no API calls so it can be unit-tested + * alone. Cached briefly: drawers and pickers reopen often, and a name rarely changes. + */ +const CACHE_MS = 60_000 + +const resourceTypeRowCached = createCache( + ({ workspace, name }: { workspace: string; name: string }) => + ResourceService.getResourceType({ workspace, path: name }).then( + (rt) => addResourceTypeDisplayName(rt), + () => {} + ), + { invalidateMs: CACHE_MS, maxSize: 50 } +) + +/** + * Fill `resourceTypeDisplayName` for one type, for a surface titled with a type it holds no row + * for. The name is stored with the type, so this reads the row rather than the hub. + */ +export function loadResourceTypeDisplayName(workspace: string, name: string): Promise { + return resourceTypeRowCached({ workspace, name }) +} + +/** Bumped by every failed read, so the next caller keys a fresh read rather than the rejection. */ +let failedReads = 0 + +const hubIntegrationsCached = createCache( + ({ kind }: { kind?: string; refresh: number; attempt: number }) => + IntegrationService.listHubIntegrations({ kind }).then( + (integrations) => { + setHubIntegrationDisplayNames(integrations) + return integrations + }, + (error) => { + failedReads += 1 + throw error + } + ), + { invalidateMs: CACHE_MS } +) + +/** + * The hub's integration list, read once a minute per `kind` however many pickers ask, recording + * each integration's name on the way. A failed read rejects, so a picker can say the hub is + * unavailable, but is not kept: the next caller reads again. `refresh` is a picker's refresh + * count, and a new value reads again too. + */ +export function listHubIntegrationsShared(kind?: string, refresh = 0) { + return hubIntegrationsCached({ kind, refresh, attempt: failedReads }) +} + +/** + * Fill `integrationDisplayName` for a picker whose integrations come from its own items rather + * than the hub's integration list, as the hub app and flow pickers do. Unfiltered: `kind` + * narrows by script kind, so asking for an app or a flow would name nothing. + */ +export function loadHubIntegrationDisplayNames(): Promise { + if (get(disableHubStore)) return Promise.resolve() + return listHubIntegrationsShared().then( + () => {}, + () => {} + ) +} diff --git a/frontend/src/lib/components/flows/pickers/PickHubApp.svelte b/frontend/src/lib/components/flows/pickers/PickHubApp.svelte index 234f053f7b..15177d6cd6 100644 --- a/frontend/src/lib/components/flows/pickers/PickHubApp.svelte +++ b/frontend/src/lib/components/flows/pickers/PickHubApp.svelte @@ -6,6 +6,7 @@ import NoItemFound from '$lib/components/home/NoItemFound.svelte' import RowIcon from '$lib/components/common/table/RowIcon.svelte' import { loadHubApps } from '$lib/hub' + import { loadHubIntegrationDisplayNames } from '$lib/components/displayNameLoaders' import TextInput from '$lib/components/text_input/TextInput.svelte' import { Alert } from '$lib/components/common' import { disableHubStore } from '$lib/stores' @@ -36,6 +37,7 @@ onMount(async () => { if ($disableHubStore) return + void loadHubIntegrationDisplayNames() const result = await loadHubApps() if (result === undefined) { hubNotAvailable = true diff --git a/frontend/src/lib/components/flows/pickers/PickHubFlow.svelte b/frontend/src/lib/components/flows/pickers/PickHubFlow.svelte index 1f3c659bf8..8c07ad1506 100644 --- a/frontend/src/lib/components/flows/pickers/PickHubFlow.svelte +++ b/frontend/src/lib/components/flows/pickers/PickHubFlow.svelte @@ -6,6 +6,7 @@ import NoItemFound from '$lib/components/home/NoItemFound.svelte' import RowIcon from '$lib/components/common/table/RowIcon.svelte' import { loadHubFlows } from '$lib/hub' + import { loadHubIntegrationDisplayNames } from '$lib/components/displayNameLoaders' import TextInput from '$lib/components/text_input/TextInput.svelte' import { Alert } from '$lib/components/common' import { disableHubStore } from '$lib/stores' @@ -36,6 +37,7 @@ onMount(async () => { if ($disableHubStore) return + void loadHubIntegrationDisplayNames() const result = await loadHubFlows() if (result === undefined) { hubNotAvailable = true diff --git a/frontend/src/lib/components/flows/pickers/PickHubScript.svelte b/frontend/src/lib/components/flows/pickers/PickHubScript.svelte index d84228dd35..05205d8335 100644 --- a/frontend/src/lib/components/flows/pickers/PickHubScript.svelte +++ b/frontend/src/lib/components/flows/pickers/PickHubScript.svelte @@ -4,8 +4,9 @@ import { capitalize } from '$lib/utils' import NoItemFound from '$lib/components/home/NoItemFound.svelte' import { APP_TO_ICON_COMPONENT } from '$lib/components/icons' + import { listHubIntegrationsShared } from '$lib/components/displayNameLoaders' import ListFilters from '$lib/components/home/ListFilters.svelte' - import { IntegrationService, ScriptService, type HubScriptKind } from '$lib/gen' + import { ScriptService, type HubScriptKind } from '$lib/gen' import { Loader2 } from 'lucide-svelte' import TextInput from '$lib/components/text_input/TextInput.svelte' import { disableHubStore, workspaceStore } from '$lib/stores' @@ -64,7 +65,7 @@ hubNotAvailable = false // Independent reads, so they share one round trip before first paint. const [integrations, local] = await Promise.all([ - IntegrationService.listHubIntegrations({ kind: filterKind }), + listHubIntegrationsShared(filterKind), $workspaceStore ? localCountsByIntegration($workspaceStore) : {} ]) const hubPicks = Object.fromEntries(integrations.map((x) => [x.name, x.picks ?? 0])) diff --git a/frontend/src/lib/components/flows/pickers/PickHubScriptQuick.svelte b/frontend/src/lib/components/flows/pickers/PickHubScriptQuick.svelte index d32e82c702..2499c285c4 100644 --- a/frontend/src/lib/components/flows/pickers/PickHubScriptQuick.svelte +++ b/frontend/src/lib/components/flows/pickers/PickHubScriptQuick.svelte @@ -1,9 +1,6 @@
    - {#if !hideSidebar} - + {#if chat && chatState} + {#if !hideSidebar} + + {/if} + {/if} -
    diff --git a/frontend/src/lib/components/flows/conversations/FlowChatInterface.svelte b/frontend/src/lib/components/flows/conversations/FlowChatInterface.svelte index ab52d06102..22fe037728 100644 --- a/frontend/src/lib/components/flows/conversations/FlowChatInterface.svelte +++ b/frontend/src/lib/components/flows/conversations/FlowChatInterface.svelte @@ -3,19 +3,76 @@ import { MessageCircle, Loader2, Settings2 } from 'lucide-svelte' import ChatMessage from '$lib/components/chat/ChatMessage.svelte' import ChatInput from '$lib/components/chat/ChatInput.svelte' - import { FlowChatManager } from './FlowChatManager.svelte' import Modal from '$lib/components/common/modal/Modal.svelte' import SchemaForm from '$lib/components/SchemaForm.svelte' import { type DynamicInput } from '$lib/utils' + import { tick, untrack } from 'svelte' + import type { Chat, ChatState } from 'windmill-chat' interface Props { - manager: FlowChatManager + chat: Chat + chatState: ChatState deploymentInProgress?: boolean additionalInputsSchema?: Record path: string + workspace?: string } - let { manager, deploymentInProgress = false, additionalInputsSchema, path }: Props = $props() + let { + chat, + chatState, + deploymentInProgress = false, + additionalInputsSchema, + path, + workspace = undefined + }: Props = $props() + + let inputMessage = $state('') + let inputElement = $state(undefined) + let messagesContainer = $state(undefined) + let loadingOlder = false + + const busy = $derived(chatState.status === 'submitted' || chatState.status === 'streaming') + // Deriveds notify only when their value changes; `chatState` itself is a new + // object on every token, and following it would drag a reader who scrolled up + // back to the end on each one. + const messageCount = $derived(chatState.messages.length) + const conversationId = $derived(chatState.conversationId) + const loadingMessages = $derived(chatState.loadingMessages) + + // Follow the conversation: new messages and a conversation switch scroll to the + // end, older pages loaded at the top keep the viewport where it was. + $effect(() => { + messageCount + conversationId + loadingMessages + untrack(() => { + if (loadingOlder) return + tick().then(() => { + if (messagesContainer) messagesContainer.scrollTop = messagesContainer.scrollHeight + }) + }) + }) + + async function handleScroll() { + if ( + !messagesContainer || + !chatState.hasMoreMessages || + chatState.loadingMessages || + loadingOlder + ) + return + if (messagesContainer.scrollTop > 10) return + loadingOlder = true + const previousHeight = messagesContainer.scrollHeight + try { + await chat.loadOlderMessages() + await tick() + messagesContainer.scrollTop = messagesContainer.scrollHeight - previousHeight + } finally { + loadingOlder = false + } + } // Derive helperScript for dynamic inputs from schema const dynamicInputHelperScript = $derived.by((): DynamicInput.HelperScript | undefined => { @@ -63,11 +120,17 @@ showInputsModal = false } - function handleSendMessage() { + async function handleSendMessage() { + const text = inputMessage.trim() + if (!text || busy || deploymentInProgress) return const inputs = additionalInputsSchema ? (loadInputsFromStorage() ?? additionalInputsValues) : undefined - manager.sendMessage(inputs) + inputMessage = '' + // A failure is reported through the chat's `onError` and as a failed message. + await chat.sendMessage(text, { inputs }).catch(() => {}) + await tick() + inputElement?.focus() } function openInputsModal() { @@ -93,7 +156,7 @@ schema={additionalInputsSchema} bind:args={additionalInputsValues} helperScript={dynamicInputHelperScript} - workspace={manager.operatingWorkspace?.()} + {workspace} /> {#snippet actions()} @@ -104,18 +167,18 @@
    {#if deploymentInProgress} {/if} - {#if manager.isLoadingMessages} + {#if chatState.loadingMessages && chatState.messages.length === 0}
    - {:else if manager.messages.length === 0} + {:else if chatState.messages.length === 0}

    Start a conversation

    @@ -123,16 +186,15 @@
    {:else}
    - {#each manager.messages as message (message.id)} + {#each chatState.messages as message (message.id)} {/each} - {#if manager.isWaitingForResponse} + {#if busy}
    Processing... @@ -148,7 +210,7 @@
    diff --git a/frontend/src/lib/components/flows/conversations/FlowChatManager.svelte.ts b/frontend/src/lib/components/flows/conversations/FlowChatManager.svelte.ts deleted file mode 100644 index bc8033e45d..0000000000 --- a/frontend/src/lib/components/flows/conversations/FlowChatManager.svelte.ts +++ /dev/null @@ -1,713 +0,0 @@ -import type { FlowConversation, FlowConversationMessage } from '$lib/gen/types.gen' -import { FlowConversationsService, JobService } from '$lib/gen' -import { sendUserToast } from '$lib/toast' -import { waitJob } from '$lib/components/waitJob' -import { tick } from 'svelte' -import InfiniteList from '$lib/components/InfiniteList.svelte' -import { workspaceStore, userStore } from '$lib/stores' -import { get } from 'svelte/store' -import { parseStreamDeltas } from '$lib/components/chat/utils' -import { randomUUID } from '$lib/utils/uuid' - -export interface ChatMessage extends FlowConversationMessage { - loading?: boolean - streaming?: boolean -} - -export interface ConversationWithDraft extends FlowConversation { - isDraft?: boolean -} - -// Per-turn stream state, kept across SSE reconnects to the same job. -interface StreamTurnState { - accumulatedContent: string - assistantMessageId: string - // Last offset the server reported; sent back on reconnect so the stream resumes - // after the deltas already rendered rather than replaying from the start. - // It indexes the stream of `streamJobId` only. - streamOffset: number | undefined - streamJobId: string | undefined -} - -export class FlowChatManager { - // State - messages = $state([]) - inputMessage = $state('') - isLoading = $state(false) - isLoadingMessages = $state(false) - isWaitingForResponse = $state(false) - messagesContainer = $state(undefined) - inputElement = $state(undefined) - page = $state(1) - hasMoreMessages = $state(false) - loadingMoreMessages = $state(false) - currentEventSource = $state(undefined) - pollingInterval = $state | undefined>(undefined) - currentJobId = $state(undefined) - conversations = $state([]) - deletingConversationId = $state(undefined) - isSidebarExpanded = $state(false) - selectedConversationId = $state(undefined) - conversationListComponent = $state(undefined) - - // Private state - #conversationsCache = $state>({}) - #scrollTimeout: ReturnType | undefined = undefined - #perPage = 50 - - // Options - #onRunFlow?: ( - userMessage: string, - conversationId: string, - additionalInputs?: Record - ) => Promise - #useStreaming = $state(false) - #path = $state(undefined) - - // When the flow editor runs as an AI-session live editor, it acts on a workspace - // that can differ from the nav store. FlowChat.svelte wires this to - // FlowEditorContext.opWorkspace so workspace-scoped calls hit the acting workspace. - operatingWorkspace?: () => string | undefined - - #workspace(): string | undefined { - return this.operatingWorkspace?.() ?? get(workspaceStore) - } - - initialize( - onRunFlow: ( - userMessage: string, - conversationId: string, - additionalInputs?: Record - ) => Promise, - path: string, - useStreaming: boolean = false - ) { - this.#onRunFlow = onRunFlow - this.#path = path - this.#useStreaming = useStreaming - } - - updateConversationId(conversationId: string | undefined) { - this.selectedConversationId = conversationId - } - - cleanup() { - if (this.currentEventSource) { - this.currentEventSource.close() - this.currentEventSource = undefined - } - this.stopPolling() - this.isLoading = false - this.isWaitingForResponse = false - this.currentJobId = undefined - } - - // Public methods for component to call - fillInputMessage(message: string) { - this.inputMessage = message - } - - focusInput() { - this.inputElement?.focus() - } - - clearMessages() { - this.messages = [] - this.inputMessage = '' - this.page = 1 - } - - async createConversation({ clearMessages = true }: { clearMessages?: boolean }) { - // Check if there's already a draft conversation - const existingDraft = this.conversations.find((c) => c.isDraft) - if (existingDraft) { - // Select the existing draft instead of creating a new one - this.selectedConversationId = existingDraft.id - this.clearMessages() - return existingDraft.id - } - const newConversationId = randomUUID() - this.selectedConversationId = newConversationId - - // Create a new conversation object and add it to the top of the list - const newConversation: ConversationWithDraft = { - id: newConversationId, - workspace_id: this.#workspace()!, - flow_path: this.#path!, - title: 'New chat', - created_at: new Date().toISOString(), - updated_at: new Date().toISOString(), - created_by: get(userStore)!.username!, - isDraft: true - } - - // Prepend to conversations list - this.conversations = [newConversation, ...this.conversations] - // Clear messages in the chat interface - if (clearMessages) { - this.clearMessages() - } - this.focusInput() - - return newConversationId - } - - setupInfiniteList() { - this.conversationListComponent?.setLoader((page, perPage) => - this.loadConversations(page, perPage) - ) - this.conversationListComponent?.setDeleteItemFn((id) => this.deleteConversation(id)) - } - - async selectConversation(conversationId: string, isDraft?: boolean) { - this.selectedConversationId = conversationId - // Load conversation messages into chat interface - if (isDraft) { - // For draft conversations, just clear messages (don't try to load from backend) - this.clearMessages() - } else { - // For persisted conversations, load messages from backend - await this.loadConversationMessages(conversationId) - } - } - - async refreshConversations() { - await this.conversationListComponent?.loadData('forceRefresh') - } - - // Only used by InfiniteList - private async deleteConversation(conversationId: string) { - try { - this.deletingConversationId = conversationId - await FlowConversationsService.deleteFlowConversation({ - workspace: this.#workspace()!, - conversationId - }) - if (this.selectedConversationId === conversationId) { - this.selectedConversationId = undefined - this.clearMessages() - } - sendUserToast('Conversation deleted successfully') - } catch (error) { - console.error('Failed to delete conversation:', error) - sendUserToast('Failed to delete conversation', true) - throw error - } finally { - this.deletingConversationId = undefined - } - } - - async cancelCurrentJob() { - if (!this.#workspace()) { - return - } - - try { - if (this.currentJobId) { - await JobService.cancelQueuedJob({ - workspace: this.#workspace()!, - id: this.currentJobId, - requestBody: {} - }) - sendUserToast(`Job ${this.currentJobId} cancelled`) - } - } catch (error) { - console.error('Error cancelling job:', error) - sendUserToast('Could not cancel job', true) - } finally { - this.cleanup() - } - } - - async loadConversationMessages(conversationId?: string) { - this.page = 1 - await this.loadMessages(true, conversationId) - } - - // Only used by InfiniteList - private async loadConversations(page: number, perPage: number) { - if (!this.#workspace() || !this.#path) return [] - - try { - const response = await FlowConversationsService.listFlowConversations({ - workspace: this.#workspace()!, - flowPath: this.#path, - page: page, - perPage: perPage - }) - - return response - } catch (error) { - console.error('Failed to load conversations:', error) - sendUserToast('Failed to load conversations', true) - return [] - } - } - - // Message loading - private async loadMessages(reset: boolean, conversationId?: string) { - let conversationIdToUse = conversationId ?? this.selectedConversationId - if (!this.#workspace() || !conversationIdToUse) return - - if (reset) { - if (this.#conversationsCache[conversationIdToUse]) { - this.messages = this.#conversationsCache[conversationIdToUse] - return - } - this.isLoadingMessages = true - } else { - this.loadingMoreMessages = true - } - - const pageToFetch = reset ? 1 : this.page + 1 - - try { - const previousScrollHeight = this.messagesContainer?.scrollHeight || 0 - - const response = await FlowConversationsService.listConversationMessages({ - workspace: this.#workspace()!, - conversationId: conversationIdToUse, - page: pageToFetch, - perPage: this.#perPage - }) - - if (reset) { - this.#conversationsCache[conversationIdToUse] = response - this.messages = response - this.isLoadingMessages = false - await new Promise((resolve) => setTimeout(resolve, 100)) - this.scrollToBottom() - } else { - this.messages = [...response, ...this.messages] - this.page = pageToFetch - // Restore scroll position - await new Promise((resolve) => setTimeout(resolve, 50)) - if (this.messagesContainer) { - this.messagesContainer.scrollTop = - this.messagesContainer.scrollHeight - previousScrollHeight - } - } - - this.hasMoreMessages = response.length === this.#perPage - } catch (error) { - console.error('Failed to load messages:', error) - sendUserToast('Failed to load messages: ' + error) - } finally { - this.isLoadingMessages = false - this.loadingMoreMessages = false - } - } - - handleScroll = () => { - if (this.#scrollTimeout) clearTimeout(this.#scrollTimeout) - - this.#scrollTimeout = setTimeout(() => { - if (!this.messagesContainer || !this.hasMoreMessages || this.loadingMoreMessages) return - - if (this.messagesContainer.scrollTop <= 10) { - this.loadMessages(false) - } - }, 200) - } - - scrollToBottom() { - if (this.messagesContainer) { - this.messagesContainer.scrollTop = this.messagesContainer.scrollHeight - } - } - - private scrollToUserMessage(messageId: string) { - if (!this.messagesContainer) return - const messageElement = this.messagesContainer.querySelector(`[data-message-id="${messageId}"]`) - if (messageElement) { - messageElement.scrollIntoView({ behavior: 'smooth', block: 'start' }) - } - } - - private getLastPersistedMessageSeq() { - for (let i = this.messages.length - 1; i >= 0; i--) { - const message = this.messages[i] - if (!message.id.startsWith('temp-')) { - return message.created_seq - } - } - - return undefined - } - - // Polling - private async pollJobResult(jobId: string) { - try { - await waitJob(jobId, this.#workspace()) - } catch (error) { - console.error('Error polling job result:', error) - } finally { - // Do a final poll to get all messages from database - try { - if (this.selectedConversationId) { - await this.pollConversationMessages(this.selectedConversationId, { - removeTempMessages: true - }) - } - } catch {} - this.cleanup() - } - } - - private async pollConversationMessages( - conversationId: string, - options?: { isNewConversation?: boolean; removeTempMessages?: boolean } - ) { - if (!this.#workspace()) return - - try { - const lastSeq = this.getLastPersistedMessageSeq() - const response = await FlowConversationsService.listConversationMessages({ - workspace: this.#workspace()!, - conversationId: conversationId, - page: 1, - perPage: 50, - afterSeq: lastSeq - }) - - if (options?.isNewConversation) { - await this.refreshConversations() - } - - const filteredResponse = response.filter((msg) => msg.message_type !== 'user') - for (const msg of filteredResponse) { - if (!this.messages.find((m) => m.id === msg.id)) { - this.messages = [...this.messages, msg] - } - } - - // Only remove temporary messages when explicitly requested (e.g., after job completion) - // During streaming, we keep temp messages to avoid them disappearing due to race conditions - if (options?.removeTempMessages) { - this.messages = this.messages.filter( - (msg) => !msg.id.startsWith('temp-') || msg.message_type === 'user' - ) - } - } catch (error) { - console.error('Polling error:', error) - } - } - - private startPolling(conversationId: string, isNewConversation?: boolean) { - if (this.pollingInterval) return - this.pollingInterval = setInterval(() => { - this.pollConversationMessages(conversationId, { isNewConversation }) - }, 500) // Poll every 0.5 seconds - setTimeout( - () => { - this.stopPolling() - }, - 2 * 60 * 1000 - ) // Stop polling after 2 minutes - } - - private stopPolling() { - if (this.pollingInterval) { - clearInterval(this.pollingInterval) - this.pollingInterval = undefined - } - } - - // Message sending - async sendMessage(additionalInputs?: Record) { - if (!this.inputMessage.trim() || this.isLoading) return - - const isNewConversation = this.messages.length === 0 - - // Reset state for new message - this.stopPolling() - - // Generate a new conversation ID if we don't have one - let currentConversationId = this.selectedConversationId - if (!this.selectedConversationId) { - const newConversationId = await this.createConversation({ clearMessages: false }) - currentConversationId = newConversationId - } - - if (!currentConversationId) { - console.error('No conversation ID found') - return - } - - // Invalidate the conversation cache - delete this.#conversationsCache[currentConversationId] - - const userMessage: ChatMessage = { - id: `temp-${randomUUID()}`, - content: this.inputMessage.trim(), - created_at: new Date().toISOString(), - created_seq: 0, - message_type: 'user', - conversation_id: currentConversationId - } - - this.messages = [...this.messages, userMessage] - const messageContent = this.inputMessage.trim() - this.inputMessage = '' - this.isLoading = true - this.isWaitingForResponse = true - - try { - await tick() - this.scrollToUserMessage(userMessage.id) - - if (this.#useStreaming && this.#path) { - await this.handleStreamingMessage( - messageContent, - currentConversationId, - isNewConversation, - additionalInputs - ) - } else { - await this.handlePollingMessage( - messageContent, - currentConversationId, - isNewConversation, - additionalInputs - ) - } - } catch (error) { - console.error('Error running flow:', error) - sendUserToast('Failed to run flow: ' + error, true) - } finally { - if (!this.#useStreaming) { - this.isLoading = false - } - } - - await tick() - this.focusInput() - } - - private async handleStreamingMessage( - messageContent: string, - currentConversationId: string, - isNewConversation: boolean, - additionalInputs?: Record - ) { - // Close any existing EventSource - if (this.currentEventSource) { - this.currentEventSource.close() - } - - try { - const jobId = await this.#onRunFlow?.(messageContent, currentConversationId, additionalInputs) - if (!jobId) { - console.error('No jobId returned from onRunFlow') - return - } - this.currentJobId = jobId - - this.startPolling(currentConversationId, isNewConversation) - - this.#followJob(jobId, currentConversationId, { - accumulatedContent: '', - assistantMessageId: '', - streamOffset: undefined, - streamJobId: undefined - }) - } catch (error) { - console.error('Stream connection error:', error) - sendUserToast('Failed to connect to stream', true) - this.cleanup() - } - } - - // Opens an SSE connection on an already-running job. The server closes every - // stream after TIMEOUT_SSE_STREAM, so a timeout re-enters here with the same - // job and turn state rather than starting a new run. - #followJob(jobId: string, currentConversationId: string, turn: StreamTurnState) { - const streamUrl = `/api/w/${this.#workspace()}/jobs_u/getupdate_sse/${jobId}` - const url = new URL(streamUrl, window.location.origin) - url.searchParams.set('poll_delay_ms', '50') - url.searchParams.set('fast', 'true') - url.searchParams.set('only_result', 'true') - if (turn.streamOffset !== undefined) { - url.searchParams.set('stream_offset', turn.streamOffset.toString()) - } - const eventSource = new EventSource(url.toString()) - this.currentEventSource = eventSource - let isCompleted = false - - eventSource.onmessage = async (event) => { - try { - const data = JSON.parse(event.data) - const type = data.type - - if (type === 'timeout') { - eventSource.close() - this.currentEventSource = undefined - this.#followJob(jobId, currentConversationId, turn) - return - } - - // Handle ping - just ignore - if (type === 'ping') { - return - } - - // Handle error - if (type === 'error') { - eventSource.close() - this.currentEventSource = undefined - console.error('SSE error:', data) - sendUserToast('Stream error: ' + (data.error || 'Unknown error'), true) - this.cleanup() - return - } - - // Handle not found - if (type === 'not_found') { - eventSource.close() - this.currentEventSource = undefined - console.error('Job not found') - sendUserToast('Job not found', true) - this.cleanup() - return - } - - if (type === 'update') { - if (data.flow_stream_job_id) { - this.currentJobId = data.flow_stream_job_id - if (data.flow_stream_job_id !== turn.streamJobId) { - const offsetFromOtherJob = - turn.streamJobId !== undefined && turn.streamOffset !== undefined - turn.streamJobId = data.flow_stream_job_id - if (offsetFromOtherJob) { - // The offset indexes the previous sub-job's stream (a retried last step - // gets a new one), so this connection skipped the new job's first chunks. - // Drop this delta and re-attach from the start of the new sub-job. - turn.streamOffset = undefined - eventSource.close() - this.currentEventSource = undefined - this.#followJob(jobId, currentConversationId, turn) - return - } - } - } - if (data.stream_offset !== undefined) { - turn.streamOffset = data.stream_offset - } - // Process new stream content - if (data.new_result_stream) { - // Stop polling since we are receiving last step streaming - this.stopPolling() - const { type, content: newContent, success } = parseStreamDeltas(data.new_result_stream) - turn.accumulatedContent += newContent - - // Create tool message if type is tool_result - if (type === 'tool_result') { - // set last message streaming to false - this.messages = this.messages.map((msg) => - msg.id === this.messages[this.messages.length - 1].id - ? { ...msg, streaming: false } - : msg - ) - - this.messages = [ - ...this.messages, - { - id: 'temp-' + randomUUID(), - content: newContent, - created_at: new Date().toISOString(), - created_seq: 0, - message_type: 'tool', - conversation_id: currentConversationId, - job_id: '', - loading: false, - streaming: false, - success - } - ] - // Reset assistant message ID since we are creating a tool message - turn.assistantMessageId = '' - turn.accumulatedContent = '' - } - - // Create message on first content - else if ( - type === 'message' && - turn.assistantMessageId.length === 0 && - turn.accumulatedContent.length > 0 - ) { - turn.assistantMessageId = 'temp-' + randomUUID() - this.messages = [ - ...this.messages, - { - id: turn.assistantMessageId, - content: turn.accumulatedContent, - created_at: new Date().toISOString(), - created_seq: 0, - message_type: 'assistant', - conversation_id: currentConversationId, - job_id: '', - loading: false, - streaming: true - } - ] - } else { - // Update existing message - this.messages = this.messages.map((msg) => - msg.id === turn.assistantMessageId - ? { ...msg, content: turn.accumulatedContent } - : msg - ) - } - } - - // Handle completion - if (data.completed) { - isCompleted = true - // Do a final poll to get all messages from database - if (this.selectedConversationId) { - await this.pollConversationMessages(this.selectedConversationId, { - removeTempMessages: true - }) - } - this.cleanup() - } - } - } catch (error) { - console.error('Error processing stream event:', error) - } - } - - eventSource.onerror = (error) => { - if (isCompleted) return - console.error('EventSource error:', error) - sendUserToast('Stream error occurred', true) - this.cleanup() - } - } - - private async handlePollingMessage( - messageContent: string, - currentConversationId: string, - isNewConversation: boolean, - additionalInputs?: Record - ) { - const jobId = await this.#onRunFlow?.(messageContent, currentConversationId, additionalInputs) - if (!jobId) { - console.error('No jobId returned from onRunFlow') - return - } - - // Store the current job ID so it can be cancelled - this.currentJobId = jobId - - if (isNewConversation) { - await this.refreshConversations() - } - - // Start polling for intermediate messages in non-streaming mode too - this.startPolling(currentConversationId) - this.pollJobResult(jobId) - } -} - -export const createFlowChatManager = () => new FlowChatManager() diff --git a/frontend/src/lib/components/flows/conversations/FlowConversationsSidebar.svelte b/frontend/src/lib/components/flows/conversations/FlowConversationsSidebar.svelte index 2f09bcebe1..02c2581db1 100644 --- a/frontend/src/lib/components/flows/conversations/FlowConversationsSidebar.svelte +++ b/frontend/src/lib/components/flows/conversations/FlowConversationsSidebar.svelte @@ -1,26 +1,72 @@
    @@ -31,11 +77,11 @@ unifiedSize="md" variant="subtle" startIcon={{ - icon: manager.isSidebarExpanded ? PanelLeftClose : PanelLeftOpen, + icon: expanded ? PanelLeftClose : PanelLeftOpen, classes: 'ml-[2px]' }} - onClick={() => (manager.isSidebarExpanded = !manager.isSidebarExpanded)} - iconOnly={!manager.isSidebarExpanded} + onClick={() => (expanded = !expanded)} + iconOnly={!expanded} btnClasses={'justify-start transition-all duration-150'} title="Conversations" > @@ -45,9 +91,9 @@ unifiedSize="md" variant="subtle" startIcon={{ icon: Plus, classes: 'ml-[2px]' }} - onClick={() => manager.createConversation({ clearMessages: true })} + onClick={newChat} title="Start new conversation" - iconOnly={!manager.isSidebarExpanded} + iconOnly={!expanded} btnClasses={'justify-start transition-all duration-150 whitespace-nowrap'} >
    New chat
    @@ -56,50 +102,69 @@
    - {#if !manager.isSidebarExpanded} + {#if !expanded}
    {/if} -
    +
    + {#if draftShown && expanded} +
    + +
    + {/if} - {#snippet customRow({ item: conversation, hover })} - {#if manager.isSidebarExpanded} + {#snippet customRow({ item: conversation })} + {#if expanded}
    diff --git a/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte b/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte index 441ed52373..6a6e731c11 100644 --- a/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte @@ -84,7 +84,6 @@ onEditInForkClick } from '$lib/utils/editInFork' import { isCloudHosted } from '$lib/cloud' - import { agentStreamingEnabled } from '$lib/components/flows/agentFormFields' let flow: Flow | undefined = $state() let can_write = $state(false) @@ -523,12 +522,6 @@ let showEditButtons = $state(false) let mainButtons = $derived(getMainButtons(flow, args)) let chatInputEnabled = $derived(flow?.value?.chat_input_enabled ?? false) - let shouldUseStreaming = $derived.by(() => { - const modules = flow?.value?.modules - const lastModule = modules && modules.length > 0 ? modules[modules.length - 1] : undefined - if (lastModule?.value?.type !== 'aiagent') return false - return agentStreamingEnabled(lastModule.value) - }) @@ -701,7 +694,6 @@ onRunFlow={runFlowForChat} {deploymentInProgress} path={flow?.path ?? ''} - useStreaming={shouldUseStreaming} inputSchema={flow?.schema} /> {:else} diff --git a/frontend/svelte.config.js b/frontend/svelte.config.js index 1752fed97c..15634551e6 100644 --- a/frontend/svelte.config.js +++ b/frontend/svelte.config.js @@ -40,7 +40,10 @@ const config = { }, alias: { $system_prompts: '../system_prompts/auto-generated', - $oauth_connect_registry: '../backend/oauth_connect.json' + $oauth_connect_registry: '../backend/oauth_connect.json', + // The flow chat runs on the published SDK's source, so the product and the + // package share one implementation (vite.config.js allows serving it). + 'windmill-chat': '../chat-sdk/src/index.ts' } }, diff --git a/frontend/vite.config.js b/frontend/vite.config.js index a95ada7880..0140c38739 100644 --- a/frontend/vite.config.js +++ b/frontend/vite.config.js @@ -1,6 +1,7 @@ import { sveltekit } from '@sveltejs/kit/vite' import { existsSync, readFileSync } from 'fs' import { fileURLToPath } from 'url' +import { searchForWorkspaceRoot } from 'vite' import mkcert from 'vite-plugin-mkcert' const file = fileURLToPath(new URL('package.json', import.meta.url)) @@ -205,6 +206,13 @@ const config = { ], port: parseInt(process.env.FRONTEND_PORT) || 3000, cors: { origin: '*' }, + // `windmill-chat` (svelte.config.js alias) lives outside the frontend root. + fs: { + allow: [ + searchForWorkspaceRoot(process.cwd()), + fileURLToPath(new URL('../chat-sdk', import.meta.url)) + ] + }, proxy: { '^/\\.well-known/.*': { target: remoteUrl, From 129c04559548cd1bcf67758ec416fb2a48e7b928 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 15 Sep 2026 16:33:47 +0200 Subject: [PATCH 129/144] fix(cli): keep the workspace color when settings are synced from git (#11144) * fix(cli): keep the workspace color when settings are synced from git Co-Authored-By: Claude Fable 5.1 * docs(cli): name the sync direction consistently in the identity-field comments Co-Authored-By: Claude Fable 5.1 * fix(cli): apply the workspace color from settings.yaml only when the file sets one Co-Authored-By: Claude Fable 5.1 --------- Co-authored-by: Claude Fable 5.1 --- cli/src/commands/sync/sync.ts | 21 +++++++ cli/src/core/settings.ts | 19 +++++-- cli/test/push_diff_convergence_unit.test.ts | 26 ++++++++- ..._workspace_settings_identity_unit.test.ts} | 55 +++++++++++++++++-- 4 files changed, 108 insertions(+), 13 deletions(-) rename cli/test/{push_workspace_settings_name_unit.test.ts => push_workspace_settings_identity_unit.test.ts} (54%) diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index 287dd6ae12..38b9e87f84 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -2542,6 +2542,21 @@ export function preservePendingScriptLocks( } } +// `sync push` never applies the workspace's display name from settings.yaml and +// applies its color only when the local file carries one (see +// pushWorkspaceSettings), so on a push the fields it would not apply must +// compare equal, or the row is listed on every run. +const isWorkspaceSettingsFile = (p: string) => + /^settings(\.[^./\\]+)?\.(yaml|json)$/.test(p); +function stripUnappliedSettingsFields(local: any, remote: any) { + delete local?.name; + delete remote?.name; + if (local?.color == null) { + delete local?.color; + delete remote?.color; + } +} + export async function compareDynFSElement( els1: DynFSElement, els2: DynFSElement | undefined, @@ -2757,6 +2772,9 @@ export async function compareDynFSElement( delete parsedV?.enabled; delete parsedM2?.enabled; } + if (isEls1Remote === false && isWorkspaceSettingsFile(k)) { + stripUnappliedSettingsFields(parsedV, parsedM2); + } if (deepEqual(parsedV, parsedM2)) { continue; } @@ -2771,6 +2789,9 @@ export async function compareDynFSElement( delete before?.enabled; delete after?.enabled; } + if (isEls1Remote === false && isWorkspaceSettingsFile(k)) { + stripUnappliedSettingsFields(after, before); + } if (deepEqual(before, after)) { continue; } diff --git a/cli/src/core/settings.ts b/cli/src/core/settings.ts index 66fc77d230..4c95090bcf 100644 --- a/cli/src/core/settings.ts +++ b/cli/src/core/settings.ts @@ -214,9 +214,15 @@ export async function pushWorkspaceSettings( } // Exclude fields that are never applied here: slack_team_id/slack_name are OAuth-only, - // and name is not applied on pull (see below), so a name-only diff stays a no-op. + // and name is never applied (see below), so a name-only diff stays a no-op. color is + // applied only when the file carries it, so an unset one leaves the comparison too. const { slack_team_id: _lst, slack_name: _lsn, name: _ln, ...comparableLocal } = localSettings; const { slack_team_id: _rst, slack_name: _rsn, name: _rn, ...comparableRemote } = settings; + const colorManaged = localSettings.color != null; + if (!colorManaged) { + delete comparableLocal.color; + delete comparableRemote.color; + } if (isSuperset(comparableLocal, comparableRemote)) { log.debug(`Workspace settings are up to date`); return; @@ -351,10 +357,9 @@ export async function pushWorkspaceSettings( }); } - // Workspace display name is intentionally not applied on pull: settings.yaml is shared - // across a repo's branches, so applying it would let one workspace's name overwrite - // another's when both sync the same repo. It stays in the file (written on push), but a - // live workspace is only renamed by its owner. + // Workspace display name is intentionally never applied by `sync push`: settings.yaml is + // shared across a repo's branches, so applying it would let one workspace's name overwrite + // another's when both sync the same repo. `sync pull` still records it. if (localSettings.mute_critical_alerts != settings.mute_critical_alerts) { log.debug(`Updating mute critical alerts...`); @@ -366,7 +371,9 @@ export async function pushWorkspaceSettings( }); } - if (localSettings.color != settings.color) { + // A color is applied only when the file carries one: `sync pull` omits the key for a + // workspace without a color, so an unset key means "not managed by git", never "clear". + if (colorManaged && localSettings.color != settings.color) { log.debug(`Updating workspace color...`); await wmill.changeWorkspaceColor({ workspace, diff --git a/cli/test/push_diff_convergence_unit.test.ts b/cli/test/push_diff_convergence_unit.test.ts index 14e248c03e..67f6d5b4dd 100644 --- a/cli/test/push_diff_convergence_unit.test.ts +++ b/cli/test/push_diff_convergence_unit.test.ts @@ -64,6 +64,7 @@ async function diff( remoteEl: Mock, skips: Record, parentOwnsScheduleEnabled?: (scheduleFilePath: string) => boolean, + isEls1Remote = false, ) { const { changes } = await compareDynFSElement( localEl as any, @@ -76,7 +77,7 @@ async function diff( false, undefined, undefined, - false, + isEls1Remote, false, parentOwnsScheduleEnabled, ); @@ -296,3 +297,26 @@ test("push: checkout inline names stay inside the flow folder", async () => { await checkoutInlineNames(join(process.cwd(), "missing.yaml")), ).toEqual({}); }); + +// A push never applies the workspace's display name and applies its color only +// when the local file carries one (see pushWorkspaceSettings), so a file that +// differs only in what would not be applied is not a push change; a pull still +// rewrites the file. +test("push: settings.yaml differing only by name or an unset color is not a change", async () => { + const remote = local({ + "settings.yaml": "name: prod\ncolor: '#ff0000'\nerror_handler: null\n", + }); + const unsetColor = local({ + "settings.yaml": "name: staging\nerror_handler: null\n", + }); + const skips = { includeSettings: true }; + expect(await diff(unsetColor, remote, skips)).toEqual([]); + expect(await diff(remote, unsetColor, skips, undefined, true)).toEqual([ + "edited settings.yaml", + ]); + + const otherColor = local({ + "settings.yaml": "name: staging\ncolor: '#00ff00'\nerror_handler: null\n", + }); + expect(await diff(otherColor, remote, skips)).toEqual(["edited settings.yaml"]); +}); diff --git a/cli/test/push_workspace_settings_name_unit.test.ts b/cli/test/push_workspace_settings_identity_unit.test.ts similarity index 54% rename from cli/test/push_workspace_settings_name_unit.test.ts rename to cli/test/push_workspace_settings_identity_unit.test.ts index ef76ab7aec..c583a53282 100644 --- a/cli/test/push_workspace_settings_name_unit.test.ts +++ b/cli/test/push_workspace_settings_identity_unit.test.ts @@ -1,23 +1,32 @@ /** - * Regression guard: a pull (pushWorkspaceSettings) must not apply the workspace - * display name from settings.yaml. Rationale lives at the apply site in settings.ts. + * Regression guard: `sync push` (pushWorkspaceSettings) must never apply the + * workspace display name from settings.yaml, and must apply the color only when + * the file carries one. Rationale lives at the apply sites in settings.ts. */ import { expect, test, describe, beforeEach, mock } from "bun:test"; let changeWorkspaceNameCalls: unknown[] = []; +let changeWorkspaceColorCalls: unknown[] = []; let editWebhookCalls: unknown[] = []; let remoteName = ""; +let remoteColor: string | undefined = undefined; let remoteWebhook: string | undefined = undefined; // Every wmill.* call reachable from pushWorkspaceSettings is stubbed so the -// function runs without a backend; only the two we assert on record calls. +// function runs without a backend; only the three we assert on record calls. mock.module("../gen/services.gen.ts", () => ({ - getSettings: async (_a: { workspace: string }) => ({ webhook: remoteWebhook }), + getSettings: async (_a: { workspace: string }) => ({ + webhook: remoteWebhook, + color: remoteColor, + }), getWorkspaceName: async (_a: { workspace: string }) => remoteName, changeWorkspaceName: async (a: unknown) => { changeWorkspaceNameCalls.push(a); }, + changeWorkspaceColor: async (a: unknown) => { + changeWorkspaceColorCalls.push(a); + }, editWebhook: async (a: unknown) => { editWebhookCalls.push(a); }, @@ -30,7 +39,6 @@ mock.module("../gen/services.gen.ts", () => ({ editWorkspaceDefaultApp: async () => {}, editDefaultScripts: async () => {}, workspaceMuteCriticalAlertsUi: async () => {}, - changeWorkspaceColor: async () => {}, updateOperatorSettings: async () => {}, editDataTableConfig: async () => {}, editSlackCommand: async () => {}, @@ -40,13 +48,15 @@ mock.module("../gen/services.gen.ts", () => ({ const { pushWorkspaceSettings } = await import("../src/core/settings.ts"); -describe("pushWorkspaceSettings workspace name", () => { +describe("pushWorkspaceSettings workspace identity", () => { const ws = "phoenix"; beforeEach(() => { changeWorkspaceNameCalls = []; + changeWorkspaceColorCalls = []; editWebhookCalls = []; remoteName = "phoenix"; + remoteColor = undefined; remoteWebhook = undefined; }); @@ -69,4 +79,37 @@ describe("pushWorkspaceSettings workspace name", () => { expect(editWebhookCalls.length).toBe(0); expect(changeWorkspaceNameCalls.length).toBe(0); }); + + test("a settings.yaml without a color key does not clear the workspace color", async () => { + remoteColor = "#ff0000"; + remoteWebhook = "https://old"; + await pushWorkspaceSettings(ws, "settings", undefined, { + name: "phoenix", + webhook: "https://new", + }); + expect(editWebhookCalls.length).toBe(1); + expect(changeWorkspaceColorCalls.length).toBe(0); + }); + + test("a color in settings.yaml is applied when it differs from the workspace", async () => { + remoteColor = "#ff0000"; + await pushWorkspaceSettings(ws, "settings", undefined, { + name: "phoenix", + color: "#00ff00", + }); + expect(editWebhookCalls.length).toBe(0); + expect(changeWorkspaceColorCalls).toEqual([ + { workspace: ws, requestBody: { color: "#00ff00" } }, + ]); + }); + + test("a color matching the workspace is a complete no-op", async () => { + remoteColor = "#ff0000"; + await pushWorkspaceSettings(ws, "settings", undefined, { + name: "phoenix", + color: "#ff0000", + }); + expect(editWebhookCalls.length).toBe(0); + expect(changeWorkspaceColorCalls.length).toBe(0); + }); }); From 781b5a57e81eb721d97d7b87e23dd84f23895400 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 15 Sep 2026 16:36:16 +0200 Subject: [PATCH 130/144] fix(apps): run-mode inline app component uses only pinned content (#11135) Co-authored-by: Claude Opus 4.8 --- backend/tests/app_run_mode_lock_strip.rs | 240 +++++++++++++++++++++++ backend/windmill-api/src/apps.rs | 52 +++++ 2 files changed, 292 insertions(+) create mode 100644 backend/tests/app_run_mode_lock_strip.rs diff --git a/backend/tests/app_run_mode_lock_strip.rs b/backend/tests/app_run_mode_lock_strip.rs new file mode 100644 index 0000000000..e2bbf631db --- /dev/null +++ b/backend/tests/app_run_mode_lock_strip.rs @@ -0,0 +1,240 @@ +//! Regression: run mode of `execute_component`'s no-id inline-`raw_code` arm runs +//! *only* the `rawscript/`-pinned `content`, dropping the caller `hash`, +//! `lock`, `modules` and `dedicated_worker` and deriving `path` server-side — +//! all of which would otherwise run or install unpinned code as the app identity. +//! Preview mode keeps honoring the caller's fields. + +use serde_json::json; +use sha2::{Digest, Sha256}; +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(b: reqwest::RequestBuilder, token: &str) -> reqwest::RequestBuilder { + b.header("Authorization", format!("Bearer {}", token)) +} + +const CONTENT: &str = "print('benign')\n"; +// A caller lock whose presence is the whole point: if it reaches the job, the +// worker installs it. The value only needs to be recognizable in `v2_job`. +const CALLER_LOCK: &str = "evilpkg @ file:///tmp/attacker-controlled-sdist"; +// A non-codebase-sentinel hash: if it reaches the job as `runnable_id`, the +// worker fetches (and runs) a deployed script by hash instead of the pinned +// content. It need not resolve to a real row — the guard is that it never +// becomes `runnable_id`. +const CALLER_HASH: i64 = 123456789; +// A caller path in someone else's namespace: if it reaches the job as +// `runnable_path` it redirects where the pinned content's relative imports +// resolve. Run mode must instead derive the path from `/`. +const CALLER_PATH: &str = "u/attacker/evil/comp"; + +/// The pin key `execute_component` computes for a no-id inline script: +/// `rawscript/`. +fn rawscript_pin(content: &str) -> String { + let mut h = Sha256::new(); + h.update(content); + format!("rawscript/{:x}", h.finalize()) +} + +fn inline_raw_code(hash: Option, dedicated: bool) -> serde_json::Value { + let mut rc = json!({ + "language": "python3", + "content": CONTENT, + "path": CALLER_PATH, + "lock": CALLER_LOCK, + "modules": { + "m.py": { "content": "print('x')\n", "language": "python3", "lock": CALLER_LOCK } + } + }); + if let Some(h) = hash { + rc["hash"] = json!(h); + } + if dedicated { + rc["dedicated_worker"] = json!(true); + } + rc +} + +/// Fetch `(raw_lock, args-has-_MODULES, runnable_id, tag, runnable_path)` for an +/// enqueued job. +async fn job_fields( + db: &Pool, + uuid: uuid::Uuid, +) -> anyhow::Result<(Option, bool, Option, String, Option)> { + Ok(sqlx::query_as( + "SELECT raw_lock, (args ? '_MODULES'), runnable_id, tag, runnable_path \ + FROM v2_job WHERE id = $1", + ) + .bind(uuid) + .fetch_one(db) + .await?) +} + +#[sqlx::test(fixtures("base"))] +async fn test_run_mode_strips_caller_lock_and_modules(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = format!("http://localhost:{port}/api/w/test-workspace"); + + let app_path = "u/test-user/lockstrip"; + let pin = format!("comp:{}", rawscript_pin(CONTENT)); + + // Deployed Viewer-mode app whose only runnable is an inline script pinned by + // content hash and with no `app_script` row — the legacy `rawscript/` + // case that reaches the no-id run-mode arm this fix touches. + let resp = authed(client().post(format!("{ws}/apps/create")), "SECRET_TOKEN") + .json(&json!({ + "path": app_path, + "summary": "", + "value": {}, + "policy": { + "execution_mode": "viewer", + "triggerables_v2": { pin: { "static_inputs": {}, "one_of_inputs": {} } } + } + })) + .send() + .await?; + assert_eq!(resp.status(), 201, "create app: {}", resp.text().await?); + + // Run mode (no `force_viewer_static_fields`): the pin authorizes the run, but + // every caller field that selects, installs, or routes code — hash, lock, + // modules, dedicated_worker — must be dropped. + let resp = authed( + client().post(format!("{ws}/apps_u/execute_component/{app_path}")), + "SECRET_TOKEN_2", + ) + .json(&json!({ + // The args map is the other injection channel: an inline run is a + // `JobKind::Preview` job, so the worker/executors read `_MODULES` and + // `_TEMP_SCRIPT_REFS` back out of the job args. Both must be stripped. + "args": { + "_MODULES": { "m.py": { "content": "print('evil')\n", "language": "python3" } }, + "_TEMP_SCRIPT_REFS": { "../evil": "deadbeef" } + }, + "component": "comp", + "raw_code": inline_raw_code(Some(CALLER_HASH), true) + })) + .send() + .await?; + let status = resp.status(); + let body = resp.text().await?; + assert_eq!( + status, 200, + "run-mode pinned inline run must be accepted: {body}" + ); + let uuid = uuid::Uuid::parse_str(body.trim())?; + let (raw_lock, has_modules, runnable_id, tag, runnable_path) = job_fields(&db, uuid).await?; + assert_eq!( + raw_lock, None, + "run mode must strip the caller-supplied lock" + ); + assert!( + !has_modules, + "run mode must strip caller modules (both `raw_code.modules` and an `_MODULES` arg)" + ); + let has_temp_refs: bool = + sqlx::query_scalar("SELECT (args ? '_TEMP_SCRIPT_REFS') FROM v2_job WHERE id = $1") + .bind(uuid) + .fetch_one(&db) + .await?; + assert!( + !has_temp_refs, + "run mode must strip a caller `_TEMP_SCRIPT_REFS` arg (relative-import redirect)" + ); + assert_eq!( + runnable_id, None, + "run mode must strip the caller-supplied hash (no substituting a deployed script by hash)" + ); + assert!( + !tag.starts_with("dedi:"), + "run mode must strip caller `dedicated_worker` (no routing to a path-keyed dedicated worker), got tag {tag:?}" + ); + assert_eq!( + runnable_path.as_deref(), + Some(format!("{app_path}/comp").as_str()), + "run mode must derive the path server-side, not trust the caller's (relative-import base)" + ); + + // Preview mode (editor): the caller runs their own code as themselves, so the + // lock and modules are honored — the `/jobs/run/preview`-equivalent path. + let resp = authed( + client().post(format!("{ws}/apps_u/execute_component/{app_path}")), + "SECRET_TOKEN_2", + ) + .json(&json!({ + "args": {}, + "component": "comp", + "raw_code": inline_raw_code(None, false), + "force_viewer_static_fields": {} + })) + .send() + .await?; + let status = resp.status(); + let body = resp.text().await?; + assert_eq!(status, 200, "preview must be accepted: {body}"); + let uuid = uuid::Uuid::parse_str(body.trim())?; + let (raw_lock, has_modules, _, _, _) = job_fields(&db, uuid).await?; + assert_eq!( + raw_lock.as_deref(), + Some(CALLER_LOCK), + "preview must keep the caller-supplied lock" + ); + assert!(has_modules, "preview must keep the caller-supplied modules"); + + Ok(()) +} + +/// A bare `rawscript/` policy key (no `:` prefix, as `empty_triggerables` +/// migrates v1 policies) matches for any `component`, so run mode must not let a +/// path-traversing `component` steer the server-derived `runnable_path`. +#[sqlx::test(fixtures("base"))] +async fn test_run_mode_rejects_traversal_component(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = format!("http://localhost:{port}/api/w/test-workspace"); + + let app_path = "u/test-user/lockstrip_bare"; + // Bare key: no `comp:` prefix, so the pin matches regardless of `component`. + let resp = authed(client().post(format!("{ws}/apps/create")), "SECRET_TOKEN") + .json(&json!({ + "path": app_path, + "summary": "", + "value": {}, + "policy": { + "execution_mode": "viewer", + "triggerables_v2": { rawscript_pin(CONTENT): { "static_inputs": {}, "one_of_inputs": {} } } + } + })) + .send() + .await?; + assert_eq!(resp.status(), 201, "create app: {}", resp.text().await?); + + // A component that isn't a single plain segment steers the derived path's + // base: separators and `..` traverse, and an empty one shifts it up a level. + for bad in ["../../u/attacker/evil", "..", "a/b", ""] { + let resp = authed( + client().post(format!("{ws}/apps_u/execute_component/{app_path}")), + "SECRET_TOKEN_2", + ) + .json(&json!({ + "args": {}, + "component": bad, + "raw_code": inline_raw_code(None, false) + })) + .send() + .await?; + let status = resp.status(); + let body = resp.text().await?; + assert_eq!( + status, 400, + "run mode must reject component {bad:?}: got {status}: {body}" + ); + } + + Ok(()) +} diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index cc4c5079f9..6af6bbc37c 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -3853,6 +3853,26 @@ fn digest(code: &str) -> String { format!("rawscript/{:x}", result) } +/// Canonical `runnable_path` for a run-mode no-id inline app component: +/// `/` — byte-for-byte what the runtime frontend sends. It is +/// the relative-import base, so the caller must not steer it: reject a `component` +/// that isn't a single non-empty path segment (empty, a separator, or `.`/`..` +/// would walk the base out of the app, and a bare `rawscript/` policy key +/// does not pin the component). +fn inline_run_path(app_path: &str, component: &str) -> Result { + if component.is_empty() + || component.contains('/') + || component.contains('\\') + || component == "." + || component == ".." + { + return Err(Error::BadRequest( + "component id must be a single non-empty path segment".to_string(), + )); + } + Ok(format!("{app_path}/{component}")) +} + async fn get_on_behalf_details_from_policy_and_authed( policy: &Policy, opt_authed: &Option, @@ -4257,6 +4277,14 @@ async fn execute_component( let resolved_delete_secs = resolve_delete_after_secs(None, policy_triggerables.delete_after_secs); + // `_MODULES` and `_TEMP_SCRIPT_REFS` are server-injected control keys (into + // `extra`) that the worker reads back for a `Preview` job — which an inline run + // is. A caller supplying them in `args` would inject module content/locks or + // redirect relative-import resolution, unpinned, as the app identity. Drop them; + // legitimate values ride in `extra`, never the request `args`. + payload.args.remove("_MODULES"); + payload.args.remove("_TEMP_SCRIPT_REFS"); + let (mut args, job_id) = build_args( policy, policy_triggerables, @@ -4294,6 +4322,7 @@ async fn execute_component( } .filter(|t| !t.is_empty()) }; + let component = payload.component.clone(); let (job_payload, tag, _runnable_on_behalf_of) = match (payload.path, payload.raw_code, payload.id) { // flow or script: @@ -4304,6 +4333,29 @@ async fn execute_component( // `app_script` table (legacy `rawscript/`-keyed triggerables). (None, Some(raw_code), None) => { let tag = resolved_inline_tag(raw_code.tag.clone()); + let raw_code = if is_preview { + // Preview (editor / `wmill app dev`): the caller runs their own + // code, like `/jobs/run/preview` — honored verbatim. + raw_code + } else { + // Run mode. Legacy back-compat only: current deploys assign an + // `app_script` id (reduce_app) and take the `Some(id)` arm; + // drop this branch once id-less deployed apps are gone. + // + // Only `content` is pinned (`rawscript/`), so keep just + // that plus `language`/`cache_ttl`, derive `path` server-side + // (`inline_run_path`), and default the rest: a caller `hash`/ + // `lock`/`modules`/`path`/`dedicated_worker` would otherwise run + // or install unpinned code as the app identity. Reconstructing + // (vs nulling) keeps a new field defaulting safe. + RawCode { + content: raw_code.content, + language: raw_code.language, + path: Some(inline_run_path(path, &component)?), + cache_ttl: raw_code.cache_ttl, + ..Default::default() + } + }; (JobPayload::Code(raw_code), tag, None) } // inline script: run mode (deployed app) with an entry in `app_script`. From 796b6e5297d8cceb842ec097f33ec1c3115058bd Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 15 Sep 2026 17:48:33 +0200 Subject: [PATCH 131/144] feat: back AI sessions up to the workspace object storage (#11116) * feat: back AI sessions up to the workspace object storage Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01SG5qEPM6Fmf7VerXS5nnWp * fix: bind the backup key to the user and pack pushes within the server caps Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01SG5qEPM6Fmf7VerXS5nnWp * fix: keep refused and unavailable marks, one mark per key, stream the flush Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01SG5qEPM6Fmf7VerXS5nnWp * fix: settle only fully sent sessions, keep removals while backups are off, cap pull bodies Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01SG5qEPM6Fmf7VerXS5nnWp * fix: bound removal marks while backups are off and stale the sync rows instead of dropping them Co-Authored-By: Claude Fable 5.1 * fix: retry a lost lock, cap nested push lists and oversized pieces, drop a stale copy of a chat that outgrew the backup Co-Authored-By: Claude Fable 5.1 * fix: cap pieces per push, size requests in UTF-8, keep a move's removal for an off workspace, disclose the restore counter Co-Authored-By: Claude Fable 5.1 * fix: file a move's removal only once the new copy landed, retire marks through the sync row Co-Authored-By: Claude Fable 5.1 * fix: keep a session marked while deletes are carried over, drop only gone sessions' marks Co-Authored-By: Claude Fable 5.1 * fix: record what a refused flush already stored, stop early when every mark is retired Co-Authored-By: Claude Fable 5.1 * fix: restore past another workspace's removal mark, file a move's removal before its row, bound the first pulled session Co-Authored-By: Claude Fable 5.1 * fix: re-key the backups on workspace key rotation, accept only base64 images, carry a delete on the sync row when its mark cannot be written Co-Authored-By: Claude Fable 5.1 * fix: durable conditional re-key of session backups, re-push on a storage switch Co-Authored-By: Claude Fable 5.1 * fix: fail a push the key rotated under, settle no session split across storages, narrow the re-key module Co-Authored-By: Claude Fable 5.1 * fix: keep a delete filed during a push, bound the pull and re-key listings Co-Authored-By: Claude Fable 5.1 * fix: bound the session listing, mark the store's own user on a write that lands after a user switch Co-Authored-By: Claude Fable 5.1 * fix: list sessions through per-session index markers, hold a session's parts back after a failed one Co-Authored-By: Claude Fable 5.1 * fix: record a rotation on every build, list a session only on the part that completes its push Co-Authored-By: Claude Fable 5.1 * fix: leave an object larger than any push writes unread Co-Authored-By: Claude Fable 5.1 * fix: read each object against its listed size, carry a dirty mark that cannot be written on the sync row Co-Authored-By: Claude Fable 5.1 * fix: note the storages the re-key walk completed on, reach another user's rows on a failed mark, read a head at its cap Co-Authored-By: Claude Fable 5.1 * fix: read the replaced key under its row lock, carry a refused dirty bump on the sync row Co-Authored-By: Claude Fable 5.1 * fix: pull a session that outgrew one answer in pages, imported only whole Co-Authored-By: Claude Fable 5.1 * fix: build a pull page from the smallest keys of the whole listing, stage each page as it lands Co-Authored-By: Claude Fable 5.1 * fix: re-record a key rotated back to, admit earlier-page images, restage over a cut-short restore Co-Authored-By: Claude Fable 5.1 * refactor: delete the backups on key rotation instead of re-keying them Co-Authored-By: Claude Fable 5.1 * fix: end a pull page before an object that grew since the listing, prune what a cut-short restore staged Co-Authored-By: Claude Fable 5.1 * fix: delete the backups before the key commits, skip a planted object whatever its listing says, lock a restore across tabs, prune stale artifact versions Co-Authored-By: Claude Fable 5.1 * fix: keep the backups under a prefix named by the key, delete the previous key's prefix after the commit Co-Authored-By: Claude Fable 5.1 * fix: name the backup prefix by a generation the rotation bumps, never write an older record over a newer one on restore Co-Authored-By: Claude Fable 5.1 * fix: retire a removal only against the storage holding the backup, restart a paged pull whose listing moved Co-Authored-By: Claude Fable 5.1 * fix: fingerprint a pull page before reading it, answer the backup generation apart from the storage identity Co-Authored-By: Claude Fable 5.1 * fix: answer needs_head for a headless session push, prune restaged pieces by id Co-Authored-By: Claude Fable 5.1 * fix: serialize a session's push and removal, open whole pushes with the head, prune only own restores Co-Authored-By: Claude Fable 5.1 * fix: require a head on a whole push, prune before the record lands, restore only under Web Locks Co-Authored-By: Claude Fable 5.1 * fix: incremental pushes ride on a listed session, removals wait for every storage holding a copy Co-Authored-By: Claude Fable 5.1 * fix: a whole push replaces the backup under a per-push token, a pull page is checked after its reads Co-Authored-By: Claude Fable 5.1 * fix: fingerprint a pull page by entity tag and version too Co-Authored-By: Claude Fable 5.1 * fix: a moved session's removal mark names the storages holding the old copy Co-Authored-By: Claude Fable 5.1 * fix: restore a workspace family together, the newest copy of a moved session winning Co-Authored-By: Claude Fable 5.1 * fix: name the marker by the session's move count, abort a family restore a listing failed in Co-Authored-By: Claude Fable 5.1 * fix: list the family again before a restored record lands, require the pull fingerprint Co-Authored-By: Claude Fable 5.1 * fix: list the whole family once per restored workspace, off members included Co-Authored-By: Claude Fable 5.1 * fix: a push split over parts, incremental too, unlists the session until its last part Co-Authored-By: Claude Fable 5.1 * fix: refuse a partial push part that names no push Co-Authored-By: Claude Fable 5.1 * fix: keep a refused bump for a session with no row yet, probe an off workspace again Co-Authored-By: Claude Fable 5.1 * fix: backfill row-carried bumps after a reload, ask an off workspace again on a timer Co-Authored-By: Claude Fable 5.1 * fix: backfill a row for its bumps only when it carries some Co-Authored-By: Claude Fable 5.1 * fix: a backfilled mark that cannot be written counts from the page's counter Co-Authored-By: Claude Fable 5.1 * docs: say an off workspace is asked again, in the mirror's comments Co-Authored-By: Claude Fable 5.1 * chore: update ee-repo-ref to 1c1dab33563c4907aff8b0da825fb66db60af82a This commit updates the EE repository reference after PR #796 was merged in windmill-ee-private. Previous ee-repo-ref: 289b477ca3fc993da06ec09b11c8f55d5e4e39c1 New ee-repo-ref: 1c1dab33563c4907aff8b0da825fb66db60af82a Automated by sync-ee-ref workflow. * fix: unlist a session while an incremental push changes more than one object Co-Authored-By: Claude Fable 5.1 --------- Co-authored-by: Claude Fable 5.1 Co-authored-by: windmill-internal-app[bot] --- backend/ee-repo-ref.txt | 2 +- ...518_ai_sessions_backup_generation.down.sql | 1 + ...91518_ai_sessions_backup_generation.up.sql | 4 + backend/summarized_schema.txt | 2 +- backend/tests/ai_sessions.rs | 1085 +++++++++++ .../src/ai_session_backups.rs | 150 ++ backend/windmill-api-workspaces/src/lib.rs | 4 +- .../windmill-api-workspaces/src/workspaces.rs | 32 + backend/windmill-api/openapi.yaml | 296 +++ backend/windmill-api/src/ai.rs | 7 + backend/windmill-api/src/ai_sessions.rs | 1316 +++++++++++++ backend/windmill-api/src/lib.rs | 2 + backend/windmill-api/src/workspaces.rs | 6 + backend/windmill-common/src/variables.rs | 7 +- backend/windmill-object-store/src/lib.rs | 4 +- docs/ai-session-backups.md | 266 +++ .../lib/components/InstanceSettings.svelte | 22 +- .../copilot/chat/HistoryManager.svelte.ts | 173 +- .../copilot/chat/artifacts/artifactsDB.ts | 105 +- .../sessions/sessionMirror.svelte.ts | 1644 +++++++++++++++++ .../components/sessions/sessionMirror.test.ts | 1449 +++++++++++++++ .../sessions/sessionMirrorPlan.test.ts | 204 ++ .../components/sessions/sessionMirrorPlan.ts | 309 ++++ .../sessions/sessionMirrorSignal.ts | 42 + .../sessions/sessionState.svelte.ts | 77 +- .../sessions/sessionStateIndexedDb.test.ts | 34 + .../workspaceSettings/AISettings.svelte | 33 +- frontend/src/lib/userScopedDb.ts | 20 +- frontend/src/lib/userScopedStorage.ts | 14 +- .../src/routes/(root)/(logged)/+layout.svelte | 11 + 30 files changed, 7280 insertions(+), 41 deletions(-) create mode 100644 backend/migrations/20260915091518_ai_sessions_backup_generation.down.sql create mode 100644 backend/migrations/20260915091518_ai_sessions_backup_generation.up.sql create mode 100644 backend/tests/ai_sessions.rs create mode 100644 backend/windmill-api-workspaces/src/ai_session_backups.rs create mode 100644 backend/windmill-api/src/ai_sessions.rs create mode 100644 docs/ai-session-backups.md create mode 100644 frontend/src/lib/components/sessions/sessionMirror.svelte.ts create mode 100644 frontend/src/lib/components/sessions/sessionMirror.test.ts create mode 100644 frontend/src/lib/components/sessions/sessionMirrorPlan.test.ts create mode 100644 frontend/src/lib/components/sessions/sessionMirrorPlan.ts create mode 100644 frontend/src/lib/components/sessions/sessionMirrorSignal.ts diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 3adfe42b2f..7acd297dac 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -ccada062c072d7b74894b63863728fd1ef9bdffd \ No newline at end of file +1c1dab33563c4907aff8b0da825fb66db60af82a diff --git a/backend/migrations/20260915091518_ai_sessions_backup_generation.down.sql b/backend/migrations/20260915091518_ai_sessions_backup_generation.down.sql new file mode 100644 index 0000000000..2662a9f709 --- /dev/null +++ b/backend/migrations/20260915091518_ai_sessions_backup_generation.down.sql @@ -0,0 +1 @@ +ALTER TABLE workspace_settings DROP COLUMN IF EXISTS ai_sessions_backup_generation; diff --git a/backend/migrations/20260915091518_ai_sessions_backup_generation.up.sql b/backend/migrations/20260915091518_ai_sessions_backup_generation.up.sql new file mode 100644 index 0000000000..a62caefa60 --- /dev/null +++ b/backend/migrations/20260915091518_ai_sessions_backup_generation.up.sql @@ -0,0 +1,4 @@ +-- Bumped by every workspace key rotation: the AI session backups in the workspace storage +-- live under a prefix named by it, so a rotation moves to a fresh prefix and the previous +-- ones can be deleted at leisure without ever touching live objects. +ALTER TABLE workspace_settings ADD COLUMN ai_sessions_backup_generation BIGINT NOT NULL DEFAULT 0; diff --git a/backend/summarized_schema.txt b/backend/summarized_schema.txt index 463afd0d33..b93a4bf6fc 100644 --- a/backend/summarized_schema.txt +++ b/backend/summarized_schema.txt @@ -234,7 +234,7 @@ workspace_protection_rule: workspace_id(char), name(char), rules(int), bypass_gr FK: (workspace_id) -> workspace(id) workspace_runnable_dependencies: flow_path(char), runnable_path(char), script_hash(bigint), runnable_is_flow(bool), workspace_id(char), app_path(char), id(bigint) FK: (app_path, workspace_id) -> app(path, workspace_id) | (flow_path, workspace_id) -> flow(path, workspace_id) -workspace_settings: workspace_id(char), slack_team_id(char), slack_name(char), slack_command_script(char), slack_email(char), customer_id(char), plan(char), webhook(text), ai_config(jsonb), large_file_storage(jsonb), git_sync(jsonb), default_app(char), default_scripts(jsonb), deploy_ui(jsonb), mute_critical_alerts(bool), color(char), operator_settings(jsonb), teams_command_script(text), teams_team_id(text), teams_team_name(text), git_app_installations(jsonb), ducklake(jsonb), slack_oauth_client_id(char), slack_oauth_client_secret(char), datatable(jsonb), teams_team_guid(text), auto_invite(jsonb), error_handler(jsonb), success_handler(jsonb), public_app_execution_limit_per_minute(int), dbt_warehouses(jsonb), guest_access_enabled(bool), guest_jwt_public_key(text), guest_jwt_jwks_url(text) +workspace_settings: workspace_id(char), slack_team_id(char), slack_name(char), slack_command_script(char), slack_email(char), customer_id(char), plan(char), webhook(text), ai_config(jsonb), large_file_storage(jsonb), git_sync(jsonb), default_app(char), default_scripts(jsonb), deploy_ui(jsonb), mute_critical_alerts(bool), color(char), operator_settings(jsonb), teams_command_script(text), teams_team_id(text), teams_team_name(text), git_app_installations(jsonb), ducklake(jsonb), slack_oauth_client_id(char), slack_oauth_client_secret(char), datatable(jsonb), teams_team_guid(text), auto_invite(jsonb), error_handler(jsonb), success_handler(jsonb), public_app_execution_limit_per_minute(int), dbt_warehouses(jsonb), guest_access_enabled(bool), guest_jwt_public_key(text), guest_jwt_jwks_url(text), ai_sessions_backup_generation(int) FK: (workspace_id) -> workspace(id) zombie_job_counter: job_id(uuid), counter(int) FK: (job_id) -> v2_job(id) diff --git a/backend/tests/ai_sessions.rs b/backend/tests/ai_sessions.rs new file mode 100644 index 0000000000..f76385d4e1 --- /dev/null +++ b/backend/tests/ai_sessions.rs @@ -0,0 +1,1085 @@ +//! The AI session backup routes (`/w/{w}/ai/sessions/*`): a browser pushes pieces of its +//! sessions into the workspace's object storage and pulls them back whole. Pinned against a +//! FilesystemStorage LFS so the test needs no object store, which also lets it read what +//! landed on disk: the objects must be ciphertext, since bucket credentials are shared far +//! more widely than a user's transcripts. +#![cfg(all(feature = "private", feature = "parquet"))] + +use serde_json::{json, Value}; +use sqlx::{Pool, Postgres}; +use windmill_common::utils::calculate_hash; +use windmill_test_utils::*; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder, token: &str) -> reqwest::RequestBuilder { + builder.header("Authorization", format!("Bearer {}", token)) +} + +async fn configure_primary_lfs(db: &Pool, root_path: &str) -> anyhow::Result<()> { + sqlx::query!( + "UPDATE workspace_settings SET large_file_storage = $1 WHERE workspace_id = $2", + json!({ + "type": "FilesystemStorage", + "root_path": root_path, + "public_resource": null, + "advanced_permissions": null + }), + "test-workspace" + ) + .execute(db) + .await?; + Ok(()) +} + +async fn list(base: &str, token: &str) -> anyhow::Result { + let resp = authed(client().get(format!("{base}/ai/sessions/list")), token) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + Ok(resp.json().await?) +} + +async fn pull(base: &str, token: &str, ids: &[&str]) -> anyhow::Result { + let resp = authed(client().post(format!("{base}/ai/sessions/pull")), token) + .json(&json!({ "ids": ids })) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + Ok(resp.json().await?) +} + +async fn push(base: &str, token: &str, body: Value) -> anyhow::Result { + Ok( + authed(client().post(format!("{base}/ai/sessions/push")), token) + .json(&body) + .send() + .await?, + ) +} + +fn copy_dir(from: &std::path::Path, to: &std::path::Path) -> std::io::Result<()> { + std::fs::create_dir_all(to)?; + for entry in std::fs::read_dir(from)? { + let entry = entry?; + let target = to.join(entry.file_name()); + if entry.path().is_dir() { + copy_dir(&entry.path(), &target)?; + } else { + std::fs::copy(entry.path(), target)?; + } + } + Ok(()) +} + +async fn rotate(base: &str, key: &str) -> anyhow::Result<()> { + let resp = authed( + client().post(format!("{base}/workspaces/encryption_key")), + "SECRET_TOKEN", + ) + .json(&json!({ "new_key": key })) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + Ok(()) +} + +/// The user's prefix on disk, `windmill_ai_sessions/{w_id}/g{generation}/{email hash}`, +/// under whichever generation is current. +fn user_root(storage_dir: &std::path::Path, email: &str) -> std::path::PathBuf { + let workspace = storage_dir.join("windmill_ai_sessions/test-workspace"); + let hash = calculate_hash(email); + std::fs::read_dir(&workspace) + .ok() + .into_iter() + .flatten() + .flatten() + .map(|entry| entry.path().join(&hash)) + .find(|path| path.exists()) + .expect("the user has backups under the current key") +} + +/// Every file under the storage root, as bytes. +fn files_under(root: &std::path::Path) -> Vec<(std::path::PathBuf, Vec)> { + let mut out = vec![]; + let mut stack = vec![root.to_path_buf()]; + while let Some(dir) = stack.pop() { + let Ok(entries) = std::fs::read_dir(&dir) else { + continue; + }; + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + stack.push(path); + } else { + out.push((path.clone(), std::fs::read(&path).unwrap_or_default())); + } + } + } + out +} + +#[sqlx::test(fixtures("base"))] +async fn test_backups_round_trip_encrypted_and_scoped_to_the_user( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let base = format!( + "http://localhost:{}/api/w/test-workspace", + server.addr.port() + ); + + // No storage configured: the browser is told to stop trying. + let listing = list(&base, "SECRET_TOKEN").await?; + assert_eq!(listing["enabled"], false); + assert_eq!(listing["sessions"], json!([])); + + let storage_dir = tempfile::tempdir()?; + configure_primary_lfs(&db, &storage_dir.path().to_string_lossy()).await?; + + let head = + json!({ "id": "s1", "workspace_id": "test-workspace", "createdAt": 1, "chatId": "c1" }); + let chat = json!({ "id": "c1", "sessionId": "s1", "title": "MARKER_PLAINTEXT_TITLE", "lastModified": 2, + "actualMessages": [], "displayMessages": [{"role": "user", "content": "hello"}] }); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ + "id": "s1", + "whole": true, + "head": head, + "chats": [{ "id": "c1", "record": chat }, { "id": "c2", "record": { "id": "c2" } }], + "images": [{ "chat_id": "c1", "id": "img1", "data_url": "data:image/png;base64,AAAA" }], + "artifacts": { "items": [], "versions": [] } + }] + }), + ) + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let pushed: Value = resp.json().await?; + assert_eq!(pushed["enabled"], true); + assert_eq!(pushed["results"], json!([{ "id": "s1" }])); + + let listing = list(&base, "SECRET_TOKEN").await?; + assert_eq!(listing["enabled"], true); + assert_eq!(listing["sessions"][0]["id"], "s1"); + + // A part more parts follow names its push, or the session would stay listed between + // the parts: one that does not is refused before anything of it lands. + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s1", "partial": true, "delete_chats": ["c1"] }] + }), + ) + .await?; + assert_eq!(resp.status(), 400, "{}", resp.text().await?); + assert_eq!( + list(&base, "SECRET_TOKEN").await?["sessions"][0]["id"], + "s1" + ); + assert_eq!( + pull(&base, "SECRET_TOKEN", &["s1"]).await?["sessions"][0]["chats"] + .as_array() + .unwrap() + .len(), + 2 + ); + + // A part with more of the session to follow lists nothing; the part that completes + // the push does, newest first. + let ids = |listing: &Value| -> Vec { + listing["sessions"] + .as_array() + .unwrap() + .iter() + .map(|s| s["id"].as_str().unwrap().to_string()) + .collect() + }; + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s2", "whole": true, "push": "p2", "opens": true, "head": { "id": "s2", "workspace_id": "test-workspace", "createdAt": 2, "chatId": "c" }, "partial": true }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + assert_eq!(ids(&list(&base, "SECRET_TOKEN").await?), vec!["s1"]); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s2", "whole": true, "push": "p2", "chats": [{ "id": "c", "record": { "id": "c" } }] }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + assert_eq!(ids(&list(&base, "SECRET_TOKEN").await?), vec!["s2", "s1"]); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "removed": ["s2"] }), + ) + .await?; + assert_eq!(resp.status(), 200); + + // A session that outgrew one answer (three chats of 12 MB against the 32 MB budget) + // comes in pages, each naming where the next picks up, and nothing is left out. + let big = "y".repeat(12 * 1024 * 1024); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s4", "whole": true, "push": "p4", "opens": true, "head": { "id": "s4", "workspace_id": "test-workspace", "createdAt": 4, "chatId": "c1" }, "partial": true }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + for cid in ["c1", "c2", "c3"] { + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s4", "whole": true, "push": "p4", "chats": [{ "id": cid, "record": { "id": cid, "big": big } }], "partial": cid != "c3" }] + }), + ) + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + } + let mut pages = vec![]; + let mut resume = json!(null); + loop { + let body = if resume.is_null() { + json!({ "ids": ["s4"] }) + } else { + json!({ "ids": ["s4"], "resume": resume }) + }; + let resp = authed( + client().post(format!("{base}/ai/sessions/pull")), + "SECRET_TOKEN", + ) + .json(&body) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let pulled: Value = resp.json().await?; + let page = pulled["sessions"][0].clone(); + assert_eq!(page["id"], "s4"); + resume = page["next"].clone(); + pages.push(page); + if resume.is_null() { + break; + } + assert!(pages.len() < 5, "a paged pull must end"); + } + assert!(pages.len() >= 2, "36 MB must not fit one answer"); + // Every page of an unchanged backup carries the same listing fingerprint; a chat added + // to the session changes it, which is what tells a browser its pages do not belong + // together any more. + let listing = pages[0]["listing"].clone(); + assert!(listing.is_string()); + assert!(pages.iter().all(|p| p["listing"] == listing)); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s4", "chats": [{ "id": "c0", "record": { "id": "c0", "n": 1 } }] }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let pulled = pull(&base, "SECRET_TOKEN", &["s4"]).await?; + assert_ne!(pulled["sessions"][0]["listing"], listing); + // So does a chat rewritten at the same size: the fingerprint takes in the entity tag, + // not only the size and a modification time the store may report coarsely. + let listing = pulled["sessions"][0]["listing"].clone(); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s4", "chats": [{ "id": "c0", "record": { "id": "c0", "n": 2 } }] }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let pulled = pull(&base, "SECRET_TOKEN", &["s4"]).await?; + assert_ne!(pulled["sessions"][0]["listing"], listing); + let mut chat_ids: Vec = pages + .iter() + .flat_map(|p| p["chats"].as_array().unwrap().iter()) + .map(|c| c["id"].as_str().unwrap().to_string()) + .collect(); + chat_ids.sort(); + assert_eq!(chat_ids, vec!["c1", "c2", "c3"]); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "removed": ["s4"] }), + ) + .await?; + assert_eq!(resp.status(), 200); + + // More objects than a page keeps listing metadata for, from a store that lists in no + // order: the pages still carry every one of them, each once. + let many = 5001; + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s5", "whole": true, "push": "p5", "opens": true, "head": { "id": "s5", "workspace_id": "test-workspace", "createdAt": 5, "chatId": "c00000" }, "partial": true }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + for start in (0..many).step_by(100) { + let chats: Vec = (start..(start + 100).min(many)) + .map(|i| json!({ "id": format!("c{i:05}"), "record": { "id": format!("c{i:05}") } })) + .collect(); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s5", "whole": true, "push": "p5", "chats": chats, "partial": start + 100 < many }] + }), + ) + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + } + let mut seen = std::collections::HashSet::new(); + let mut resume = json!(null); + let mut pages = 0; + loop { + let body = if resume.is_null() { + json!({ "ids": ["s5"] }) + } else { + json!({ "ids": ["s5"], "resume": resume }) + }; + let resp = authed( + client().post(format!("{base}/ai/sessions/pull")), + "SECRET_TOKEN", + ) + .json(&body) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let pulled: Value = resp.json().await?; + let page = &pulled["sessions"][0]; + for c in page["chats"].as_array().unwrap() { + assert!( + seen.insert(c["id"].as_str().unwrap().to_string()), + "a chat came twice" + ); + } + pages += 1; + resume = page["next"].clone(); + if resume.is_null() { + break; + } + assert!(pages < 5, "a paged pull must end"); + } + assert_eq!(pages, 2); + assert_eq!(seen.len(), many); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "removed": ["s5"] }), + ) + .await?; + assert_eq!(resp.status(), 200); + + // A head at exactly its cap round-trips: the ciphertext read back is a block larger. + let mut big_head = json!({ "id": "s3", "workspace_id": "test-workspace", "createdAt": 3, "chatId": "c", "pad": "" }); + let pad = 1024 * 1024 - serde_json::to_string(&big_head)?.len(); + big_head["pad"] = json!("x".repeat(pad)); + assert_eq!(serde_json::to_string(&big_head)?.len(), 1024 * 1024); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "sessions": [{ "id": "s3", "whole": true, "head": big_head }] }), + ) + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let pulled = pull(&base, "SECRET_TOKEN", &["s3"]).await?; + assert_eq!(pulled["sessions"][0]["head"], big_head); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "removed": ["s3"] }), + ) + .await?; + assert_eq!(resp.status(), 200); + + // An object larger than any push writes, planted with the bucket's credentials at a + // predictable key, is not read. + let planted = + user_root(storage_dir.path(), "test@windmill.dev").join("sessions/planted/head.json"); + std::fs::create_dir_all(planted.parent().unwrap())?; + std::fs::File::create(&planted)?.set_len(32 * 1024 * 1024 + 1)?; + let pulled = pull(&base, "SECRET_TOKEN", &["planted"]).await?; + assert_eq!(pulled["sessions"], json!([])); + std::fs::remove_dir_all(planted.parent().unwrap())?; + // Under a session that exists, a planted chat is skipped without buffering and without + // the page ending before it, so the pull neither balloons nor loops. + let planted_chat = + user_root(storage_dir.path(), "test@windmill.dev").join("sessions/s1/chats/planted.json"); + std::fs::File::create(&planted_chat)?.set_len(32 * 1024 * 1024 + 1)?; + let mut resume = json!(null); + let mut pages = 0; + let mut chat_ids = vec![]; + loop { + let body = if resume.is_null() { + json!({ "ids": ["s1"] }) + } else { + json!({ "ids": ["s1"], "resume": resume }) + }; + let resp = authed( + client().post(format!("{base}/ai/sessions/pull")), + "SECRET_TOKEN", + ) + .json(&body) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let pulled: Value = resp.json().await?; + assert_eq!(pulled["sessions"][0]["head"], head); + for c in pulled["sessions"][0]["chats"].as_array().unwrap() { + chat_ids.push(c["id"].as_str().unwrap().to_string()); + } + pages += 1; + resume = pulled["sessions"][0]["next"].clone(); + if resume.is_null() { + break; + } + assert!(pages < 5, "a planted object must not keep the pull going"); + } + chat_ids.sort(); + assert_eq!(chat_ids, vec!["c1", "c2"]); + std::fs::remove_file(&planted_chat)?; + + let pulled = pull(&base, "SECRET_TOKEN", &["s1", "never-pushed"]).await?; + assert_eq!(pulled["deferred"], json!([])); + let sessions = pulled["sessions"].as_array().unwrap(); + assert_eq!(sessions.len(), 1, "an id with no backup is simply absent"); + let s1 = &sessions[0]; + assert_eq!(s1["head"], head); + let mut chats = s1["chats"].as_array().unwrap().clone(); + chats.sort_by_key(|c| c["id"].as_str().unwrap().to_string()); + assert_eq!(chats[0]["record"], chat); + assert_eq!(chats[1]["id"], "c2"); + assert_eq!( + s1["images"], + json!([{ "chat_id": "c1", "id": "img1", "data_url": "data:image/png;base64,AAAA" }]) + ); + assert_eq!(s1["artifacts"], json!({ "items": [], "versions": [] })); + + // Nothing on disk carries the transcript in the clear. + let files = files_under(storage_dir.path()); + assert!( + files.len() >= 4, + "expected the pushed objects on disk, got {files:?}" + ); + for (path, bytes) in &files { + let text = String::from_utf8_lossy(bytes); + assert!( + !text.contains("MARKER_PLAINTEXT_TITLE") && !text.contains("base64,AAAA"), + "{} holds plaintext", + path.display() + ); + } + let key_paths: Vec = files + .iter() + .map(|(p, _)| { + p.strip_prefix(storage_dir.path()) + .unwrap() + .to_string_lossy() + .to_string() + }) + .collect(); + assert!( + key_paths + .iter() + .all(|p| p.starts_with("windmill_ai_sessions/test-workspace/") + && !p.contains("test@windmill.dev")), + "keys carry the workspace and never the email: {key_paths:?}" + ); + + // Another member of the workspace sees none of it. + let other = list(&base, "SECRET_TOKEN_2").await?; + assert_eq!(other["enabled"], true); + assert_eq!(other["sessions"], json!([])); + let other = pull(&base, "SECRET_TOKEN_2", &["s1"]).await?; + assert_eq!(other["sessions"], json!([])); + + // Nor after copying the first user's ciphertext under their own prefix, which anyone + // holding the bucket credentials can do: the key is bound to the user, not the workspace. + let first = user_root(storage_dir.path(), "test@windmill.dev"); + let second = first + .parent() + .unwrap() + .join(calculate_hash("test2@windmill.dev")); + copy_dir(&first, &second)?; + let other = pull(&base, "SECRET_TOKEN_2", &["s1"]).await?; + assert_eq!( + other["sessions"], + json!([]), + "relocated ciphertext must not decrypt for another user" + ); + std::fs::remove_dir_all(&second)?; + + // Deleting a chat takes its images along; a head-only push leaves the rest in place. + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s1", "delete_chats": ["c1"] }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let pulled = pull(&base, "SECRET_TOKEN", &["s1"]).await?; + let s1 = &pulled["sessions"][0]; + assert_eq!(s1["head"], head); + assert_eq!(s1["chats"].as_array().unwrap().len(), 1); + assert_eq!(s1["chats"][0]["id"], "c2"); + assert_eq!(s1["images"], json!([])); + + // Rotating the workspace key moves the routes to a fresh generation's prefix and deletes + // the older ones off the request rather than re-key anything; the answers name the new + // generation (`backup_generation`), which is what makes every browser push its sessions + // whole again, while `storage_id` names the storage and stays. + let before = list(&base, "SECRET_TOKEN").await?; + rotate(&base, &"b".repeat(64)).await?; + for _ in 0..100 { + if files_under(storage_dir.path()).is_empty() { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + assert!( + files_under(storage_dir.path()).is_empty(), + "a rotation must leave no backup object behind" + ); + let listing = list(&base, "SECRET_TOKEN").await?; + assert_eq!(listing["sessions"], json!([])); + assert_eq!(listing["storage_id"], before["storage_id"]); + assert_ne!( + listing["backup_generation"], before["backup_generation"], + "a rotation must bump the backup generation" + ); + assert_eq!( + pull(&base, "SECRET_TOKEN", &["s1"]).await?["sessions"], + json!([]) + ); + // The browser's next push fills the storage back under the new key. + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s1", "whole": true, "head": head, "chats": [{ "id": "c2", "record": { "id": "c2" } }] }] + }), + ) + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let pulled = pull(&base, "SECRET_TOKEN", &["s1"]).await?; + assert_eq!(pulled["sessions"][0]["head"], head); + assert_eq!(pulled["sessions"][0]["chats"][0]["id"], "c2"); + // Setting the key already in place is not a rotation the browsers would notice, so it + // keeps the backups. + let same = list(&base, "SECRET_TOKEN").await?; + rotate(&base, &"b".repeat(64)).await?; + let listing = list(&base, "SECRET_TOKEN").await?; + assert_eq!(listing["storage_id"], same["storage_id"]); + assert_eq!(listing["backup_generation"], same["backup_generation"]); + assert_eq!(listing["sessions"][0]["id"], "s1"); + assert_eq!( + pull(&base, "SECRET_TOKEN", &["s1"]).await?["sessions"][0]["head"], + head + ); + + // A push that does not open the session whole rides on the head in the storage; with + // none there (another device removed the backup, or nothing was ever pushed) it is + // refused and lists nothing, head or no head on it, until the session goes whole. + let s6_head = + json!({ "id": "s6", "workspace_id": "test-workspace", "createdAt": 6, "chatId": "c" }); + let not_listed = |listing: Value| { + listing["sessions"] + .as_array() + .unwrap() + .iter() + .all(|s| s["id"] != "s6") + }; + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s6", "chats": [{ "id": "c", "record": { "id": "c" } }] }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let answer: Value = resp.json().await?; + assert_eq!(answer["results"][0]["needs_whole"], true); + assert!(not_listed(list(&base, "SECRET_TOKEN").await?)); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s6", "whole": true, "head": s6_head, "chats": [{ "id": "c", "record": { "id": "c" } }] }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let answer: Value = resp.json().await?; + assert!(answer["results"][0]["needs_whole"].is_null()); + assert_eq!( + list(&base, "SECRET_TOKEN").await?["sessions"][0]["id"], + "s6" + ); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "removed": ["s6"] }), + ) + .await?; + assert_eq!(resp.status(), 200); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s6", "head": s6_head, "chats": [{ "id": "c2", "record": { "id": "c2" } }] }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let answer: Value = resp.json().await?; + assert_eq!(answer["results"][0]["needs_whole"], true); + assert!(not_listed(list(&base, "SECRET_TOKEN").await?)); + assert!( + files_under(&user_root(storage_dir.path(), "test@windmill.dev").join("sessions/s6")) + .is_empty() + ); + + // Between the parts of a whole push (head landed, marker not yet), an incremental push + // from another device is refused too: it rides on a listed session, and there is none + // until the last part, which lists it. + let s8_head = + json!({ "id": "s8", "workspace_id": "test-workspace", "createdAt": 8, "chatId": "c1" }); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s8", "whole": true, "push": "p8", "opens": true, "head": s8_head, "partial": true }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s8", "chats": [{ "id": "c9", "record": { "id": "c9" } }] }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let answer: Value = resp.json().await?; + assert_eq!(answer["results"][0]["needs_whole"], true); + let listed = |listing: Value| { + listing["sessions"] + .as_array() + .unwrap() + .iter() + .any(|s| s["id"] == "s8") + }; + assert!(!listed(list(&base, "SECRET_TOKEN").await?)); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s8", "whole": true, "push": "p8", "chats": [{ "id": "c1", "record": { "id": "c1" } }] }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + assert!(listed(list(&base, "SECRET_TOKEN").await?)); + assert_eq!( + pull(&base, "SECRET_TOKEN", &["s8"]).await?["sessions"][0]["chats"] + .as_array() + .unwrap() + .iter() + .map(|c| c["id"].as_str().unwrap().to_string()) + .collect::>(), + vec!["c1"] + ); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "removed": ["s8"] }), + ) + .await?; + assert_eq!(resp.status(), 200); + + // A whole push replaces the backup: what the storage held of the session and the new + // push does not carry (a chat deleted while the workspace was on another storage) goes. + let s9_head = + json!({ "id": "s9", "workspace_id": "test-workspace", "createdAt": 9, "chatId": "c1" }); + let s9_chats = |ids: &[&str]| -> Vec { + ids.iter() + .map(|c| json!({ "id": c, "record": { "id": c } })) + .collect() + }; + let pulled_chats = |pulled: Value| -> Vec { + pulled["sessions"][0]["chats"] + .as_array() + .unwrap() + .iter() + .map(|c| c["id"].as_str().unwrap().to_string()) + .collect() + }; + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s9", "whole": true, "head": s9_head, "chats": s9_chats(&["c1", "c2"]) }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + assert_eq!( + pulled_chats(pull(&base, "SECRET_TOKEN", &["s9"]).await?), + vec!["c1", "c2"] + ); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s9", "whole": true, "epoch": 1, "head": s9_head, "chats": s9_chats(&["c1"]) }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + assert_eq!( + pulled_chats(pull(&base, "SECRET_TOKEN", &["s9"]).await?), + vec!["c1"] + ); + // The marker carries the move count the push named, once; an incremental push at + // another count rides on nothing, one at the same count lands. + let s9_epochs = |listing: Value| -> Vec { + listing["sessions"] + .as_array() + .unwrap() + .iter() + .filter(|s| s["id"] == "s9") + .map(|s| s["epoch"].clone()) + .collect() + }; + assert_eq!( + s9_epochs(list(&base, "SECRET_TOKEN").await?), + vec![json!(1)] + ); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s9", "chats": s9_chats(&["c7"]) }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let answer: Value = resp.json().await?; + assert_eq!(answer["results"][0]["needs_whole"], true); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s9", "epoch": 1, "chats": s9_chats(&["c7"]) }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + assert_eq!( + pulled_chats(pull(&base, "SECRET_TOKEN", &["s9"]).await?), + vec!["c1", "c7"] + ); + + // An incremental push split over parts unlists the session while it is in progress (a + // pull between two parts would take a mix of old and new pieces for the backup) and + // lists it again with the last part; while one is in progress or abandoned, a push that + // is not part of it is refused, so the browser's next push of the session goes whole. + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s9", "epoch": 1, "push": "i1", "opens": true, "chats": s9_chats(&["c8"]), "partial": true }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + assert_eq!( + s9_epochs(list(&base, "SECRET_TOKEN").await?), + Vec::::new() + ); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s9", "epoch": 1, "chats": s9_chats(&["c11"]) }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let answer: Value = resp.json().await?; + assert_eq!(answer["results"][0]["needs_whole"], true); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s9", "epoch": 1, "push": "i1", "chats": s9_chats(&["c9"]) }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + assert_eq!( + s9_epochs(list(&base, "SECRET_TOKEN").await?), + vec![json!(1)] + ); + assert_eq!( + pulled_chats(pull(&base, "SECRET_TOKEN", &["s9"]).await?), + vec!["c1", "c7", "c8", "c9"] + ); + assert_eq!( + s9_epochs(list(&base, "SECRET_TOKEN").await?), + vec![json!(1)] + ); + + // Two devices pushing the session whole at once: the push that opened later replaced + // the earlier one's pieces, so the earlier one's last part is refused and lists nothing, + // and the session is listed with the later push's pieces alone. + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s9", "whole": true, "push": "t3", "opens": true, "head": s9_head, "chats": s9_chats(&["c3"]), "partial": true }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s9", "whole": true, "push": "t4", "opens": true, "head": s9_head, "chats": s9_chats(&["c4"]), "partial": true }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s9", "whole": true, "push": "t3", "chats": s9_chats(&["c5"]) }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let answer: Value = resp.json().await?; + assert_eq!(answer["results"][0]["needs_whole"], true); + assert!(list(&base, "SECRET_TOKEN").await?["sessions"] + .as_array() + .unwrap() + .iter() + .all(|s| s["id"] != "s9")); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s9", "whole": true, "push": "t4", "chats": s9_chats(&["c6"]) }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + assert_eq!( + pulled_chats(pull(&base, "SECRET_TOKEN", &["s9"]).await?), + vec!["c4", "c6"] + ); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "removed": ["s9"] }), + ) + .await?; + assert_eq!(resp.status(), 200); + + // Removal empties both prefixes. + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "removed": ["s1"] }), + ) + .await?; + assert_eq!(resp.status(), 200); + let listing = list(&base, "SECRET_TOKEN").await?; + assert_eq!(listing["sessions"], json!([])); + assert!( + listing["storage_id"].is_string(), + "an answer names its storage: {listing}" + ); + assert!( + files_under(storage_dir.path()).is_empty(), + "removal must leave no object behind" + ); + + Ok(()) +} + +#[sqlx::test(fixtures("base", "jobs_read_auth"))] +async fn test_backup_writes_are_refused_for_the_wrong_owner_token_or_id( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let base = format!( + "http://localhost:{}/api/w/test-workspace", + server.addr.port() + ); + let storage_dir = tempfile::tempdir()?; + configure_primary_lfs(&db, &storage_dir.path().to_string_lossy()).await?; + + // A push prepared for another user must not land under the caller's prefix. + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test2@windmill.dev", "sessions": [{ "id": "s1", "head": { "id": "s1" } }] }), + ) + .await?; + assert_eq!(resp.status(), 409, "{}", resp.text().await?); + + // Ids are what the server builds keys from. + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "sessions": [{ "id": "../s1", "head": { "id": "../s1" } }] }), + ) + .await?; + assert_eq!(resp.status(), 400, "{}", resp.text().await?); + + // A whole push opens with its head; one without is refused before anything of it lands, + // and nothing lists the session. + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "sessions": [{ "id": "s7", "whole": true, "chats": [{ "id": "c", "record": { "id": "c" } }] }] }), + ) + .await?; + assert_eq!(resp.status(), 400, "{}", resp.text().await?); + assert!(list(&base, "SECRET_TOKEN").await?["sessions"] + .as_array() + .unwrap() + .iter() + .all(|s| s["id"] != "s7")); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "removed": ["a/b"] }), + ) + .await?; + assert_eq!(resp.status(), 400, "{}", resp.text().await?); + + // An image is a base64 data URL, stored and served verbatim; anything JSON would have + // to escape (and so inflate past the pull budget) is refused. + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s1", "images": [{ "chat_id": "c1", "id": "i1", "data_url": "data:image/png;base64,\u{0001}\u{0001}\"" }] }] + }), + ) + .await?; + assert_eq!(resp.status(), 400, "{}", resp.text().await?); + + // Nested lists are bounded too: each entry is an object-store call. + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s1", "delete_chats": (0..1001).map(|i| format!("c{i}")).collect::>() }] + }), + ) + .await?; + assert_eq!(resp.status(), 400, "{}", resp.text().await?); + + // ...and across the whole request, not only per entry. + let sessions: Vec = (0..100) + .map(|i| { + json!({ "id": format!("s{i}"), "delete_chats": (0..50).map(|j| format!("c{j}")).collect::>() }) + }) + .collect(); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "sessions": sessions }), + ) + .await?; + assert_eq!(resp.status(), 400, "{}", resp.text().await?); + + // A pull body is a handful of ids; a large one is refused before it is parsed. + let resp = authed( + client().post(format!("{base}/ai/sessions/pull")), + "SECRET_TOKEN", + ) + .header("Content-Type", "application/json") + .body(format!("{{\"ids\":[\"{}\"]}}", "a".repeat(100_000))) + .send() + .await?; + assert_eq!(resp.status(), 413, "{}", resp.text().await?); + + // A scoped token (here `jobs:read`) is minted for something narrower than the user's + // whole assistant history. + let resp = authed( + client().get(format!("{base}/ai/sessions/list")), + "SCOPED_DENO_TOKEN", + ) + .send() + .await?; + assert_eq!(resp.status(), 403, "{}", resp.text().await?); + + assert!(files_under(storage_dir.path()).is_empty()); + Ok(()) +} diff --git a/backend/windmill-api-workspaces/src/ai_session_backups.rs b/backend/windmill-api-workspaces/src/ai_session_backups.rs new file mode 100644 index 0000000000..bbdbfbc8c9 --- /dev/null +++ b/backend/windmill-api-workspaces/src/ai_session_backups.rs @@ -0,0 +1,150 @@ +//! What the workspace key rotation and the AI session backup routes +//! (`windmill-api/src/ai_sessions.rs`) share about the backups in the workspace storage. +//! +//! The backups are ciphertext under the workspace key and live under a prefix named by a +//! generation the rotation bumps (`workspace_settings.ai_sessions_backup_generation`) in the +//! transaction that commits the new key. A rotation does not re-key them: once committed, +//! the routes read and write under the new generation's prefix and answer with its number +//! (`backup_generation`; the storage identity, `storage_id`, names the storage and does not +//! change), so every browser marks its sync state stale and pushes its sessions whole again +//! there, and every older generation, which nothing writes to any +//! more, is deleted off the request at leisure. Sessions no browser holds any more are lost, +//! which a rotation (a rare operation) accepts in exchange for having no key but the current +//! one to read with and nothing to rewrite in place. A generation is never reused, so no +//! deletion, however late, can touch live objects; a rotation that fails before its commit +//! bumps nothing and deletes nothing; two rotations racing serialize on the key row. + +use std::sync::Arc; + +use futures::TryStreamExt; +use windmill_common::error::{Error, Result}; +use windmill_common::utils::calculate_hash; +use windmill_common::DB; +use windmill_object_store::object_store_reexports::{ + ObjectStore, ObjectStoreError, Path as ObjectPath, +}; +use windmill_object_store::{object_store_error_to_error, ObjectStoreResource}; +use windmill_types::s3::LargeFileStorage; + +/// The root of every AI session backup key in a workspace's storage. +pub const ROOT: &str = "windmill_ai_sessions"; +/// The push body cap: no object written through the routes is larger. One that is was +/// planted by whoever holds the bucket's credentials, and is left unread. +pub const MAX_OBJECT_BYTES: usize = 32 * 1024 * 1024; + +const IO_CONCURRENCY: usize = 8; + +/// The prefix of one generation's objects: `windmill_ai_sessions/{w_id}/g{generation}/`. +pub fn generation_prefix(w_id: &str, generation: i64) -> String { + format!("{ROOT}/{w_id}/g{generation}") +} + +/// Names the storage the backups are in, by what locates its objects (endpoint, region, +/// bucket; never the credentials, which rotate), so a browser tells that its sync state was +/// recorded against another storage; the generation, answered alongside, tells it a +/// rotation happened in this one. +pub fn storage_id(resource: &ObjectStoreResource) -> String { + let location = match resource { + ObjectStoreResource::S3(s) => format!( + "s3:{}:{}:{}:{}", + s.endpoint, + s.port.unwrap_or_default(), + s.region, + s.bucket + ), + ObjectStoreResource::Azure(a) => format!( + "azure:{}:{}:{}", + a.endpoint.as_deref().unwrap_or_default(), + a.account_name, + a.container_name + ), + ObjectStoreResource::Gcs(g) => format!("gcs:{}", g.bucket), + ObjectStoreResource::Filesystem(f) => format!("fs:{}", f.root_path), + }; + calculate_hash(&location)[..16].to_string() +} + +/// The workspace's primary storage, resolved without a caller: a rotation runs the +/// deletion off its own request. +async fn primary_store(db: &DB, w_id: &str) -> Result>> { + let Some(lfs_json) = sqlx::query_scalar!( + "SELECT large_file_storage FROM workspace_settings WHERE workspace_id = $1", + w_id + ) + .fetch_optional(db) + .await? + .flatten() else { + return Ok(None); + }; + let lfs: LargeFileStorage = serde_json::from_value(lfs_json) + .map_err(|e| Error::internal_err(format!("parsing large_file_storage: {e}")))?; + let resource_value = if matches!(lfs, LargeFileStorage::FilesystemStorage(_)) { + serde_json::Value::Null + } else { + let path = lfs.get_s3_resource_path(); + let path = path.strip_prefix("$res:").unwrap_or(path); + windmill_common::workspaces::transform_json_value_unchecked( + &serde_json::Value::String(format!("$res:{path}")), + w_id, + db, + ) + .await? + }; + let resource = windmill_object_store::lfs_to_object_store_resource(&lfs, resource_value)?; + Ok(Some( + windmill_object_store::build_object_store_client(&resource).await?, + )) +} + +/// The generation an object key sits under, `None` for a key of no generation (an older +/// layout), which counts as older than any. +fn generation_of(w_id: &str, key: &ObjectPath) -> Option { + key.as_ref() + .strip_prefix(&format!("{ROOT}/{w_id}/g"))? + .split('/') + .next()? + .parse() + .ok() +} + +/// Deletes, off the request and as the listing streams, every object of the workspace's +/// backups from a generation older than `current`, once the rotation that made `current` +/// the generation has committed: nothing writes there any more but a push that resolved its +/// prefix before the commit, junk the browser's next push of that session rewrites under the +/// current prefix, as is anything a deletion cut short left behind. For the rotation route, +/// which authorized its caller as a superadmin. +pub(crate) fn spawn_delete_older(db: DB, w_id: String, current: i64) { + tokio::spawn(async move { + let store = match primary_store(&db, &w_id).await { + Ok(Some(store)) => store, + Ok(None) => return, + Err(e) => { + tracing::warn!("older AI session backups of {w_id} left in place: {e:#}"); + return; + } + }; + let prefix = ObjectPath::from(format!("{ROOT}/{w_id}")); + let deleted = store + .list(Some(&prefix)) + .map_err(object_store_error_to_error) + .try_for_each_concurrent(IO_CONCURRENCY, |meta| { + let (store, w_id) = (&store, &w_id); + async move { + if generation_of(w_id, &meta.location).is_some_and(|g| g >= current) { + return Ok(()); + } + match store.delete(&meta.location).await { + Ok(()) | Err(ObjectStoreError::NotFound { .. }) => Ok(()), + Err(e) => Err(object_store_error_to_error(e)), + } + } + }) + .await; + match deleted { + Ok(()) => { + tracing::info!("deleted the AI session backups of {w_id} older than g{current}") + } + Err(e) => tracing::warn!("deleting the older AI session backups of {w_id}: {e:#}"), + } + }); +} diff --git a/backend/windmill-api-workspaces/src/lib.rs b/backend/windmill-api-workspaces/src/lib.rs index 017c9702a5..22f2a2c5bb 100644 --- a/backend/windmill-api-workspaces/src/lib.rs +++ b/backend/windmill-api-workspaces/src/lib.rs @@ -1,6 +1,8 @@ +#[cfg(feature = "parquet")] +pub mod ai_session_backups; +pub mod data_metrics; pub mod datatable_migrations; pub mod deployment_requests; -pub mod data_metrics; pub mod workspaces; pub mod workspaces_extra; pub mod workspaces_oss; diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 6048772ff6..1a3c24f096 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -5339,6 +5339,17 @@ async fn set_encryption_key( let mut tx = db.begin().await?; + // Under the row's lock, so two rotations racing serialize and each sees the key the + // other committed. The AI session backups in the workspace storage live under a prefix + // named by a generation this bumps (with the key, in this transaction) rather than + // being re-keyed; the older generations are deleted once this one has committed (see + // `ai_session_backups`). The same key set again is no rotation to them. + let previous_key: String = sqlx::query_scalar( + "SELECT key FROM workspace_key WHERE workspace_id = $1 AND kind = 'cloud' FOR UPDATE", + ) + .bind(&w_id) + .fetch_one(&mut *tx) + .await?; sqlx::query!( "UPDATE workspace_key SET key = $1 WHERE workspace_id = $2", request.new_key.clone(), @@ -5346,6 +5357,18 @@ async fn set_encryption_key( ) .execute(&mut *tx) .await?; + let backups_generation: Option = if previous_key != request.new_key { + sqlx::query_scalar( + "UPDATE workspace_settings SET ai_sessions_backup_generation = \ + ai_sessions_backup_generation + 1 WHERE workspace_id = $1 \ + RETURNING ai_sessions_backup_generation", + ) + .bind(&w_id) + .fetch_optional(&mut *tx) + .await? + } else { + None + }; let mut reencrypted_secret_paths: Vec = Vec::new(); if !request.skip_reencrypt.unwrap_or(false) { @@ -5402,6 +5425,15 @@ async fn set_encryption_key( // Invalidate the cache only after the transaction has committed WORKSPACE_CRYPT_CACHE.remove(w_id.as_str()); + // Nothing writes under the older generations any more; the browsers push their + // sessions again under the new one. + #[cfg(feature = "parquet")] + if let Some(generation) = backups_generation { + crate::ai_session_backups::spawn_delete_older(db.clone(), w_id.clone(), generation); + } + #[cfg(not(feature = "parquet"))] + let _ = backups_generation; + // Build the batch: one event for the encryption key itself plus one per // re-encrypted secret variable. The batch entrypoint dispatches a single // git-sync job per repo carrying all items, so repos with Secrets sync diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 500504a663..9f742a129b 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -13195,6 +13195,153 @@ paths: type: boolean description: more buckets matched than were returned, so summing them under-reports + /w/{workspace}/ai/sessions/list: + get: + summary: list the calling user's AI session backups in the workspace object storage + operationId: listAiSessionBackups + tags: + - ai + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: backups, newest first; `enabled` is false when the workspace has no storage for them + content: + application/json: + schema: + type: object + required: + - enabled + - sessions + properties: + enabled: + type: boolean + storage_id: + type: string + description: names the storage answered from; sync state recorded against another one is void + backup_generation: + type: integer + description: bumped by every workspace key rotation; sync state recorded under another one is void + sessions: + type: array + description: the newest 500 at most + items: + $ref: "#/components/schemas/AISessionBackupListing" + truncated: + type: boolean + description: the user has more sessions than the answer names + + /w/{workspace}/ai/sessions/pull: + post: + summary: fetch whole AI session backups + operationId: pullAiSessionBackups + tags: + - ai + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - ids + properties: + ids: + type: array + maxItems: 20 + items: + type: string + resume: + $ref: "#/components/schemas/AISessionBackupCursor" + responses: + "200": + description: the backups found; `deferred` lists ids that did not fit the response budget + content: + application/json: + schema: + type: object + required: + - enabled + - sessions + - deferred + properties: + enabled: + type: boolean + storage_id: + type: string + backup_generation: + type: integer + sessions: + type: array + items: + $ref: "#/components/schemas/AISessionBackup" + deferred: + type: array + items: + type: string + + /w/{workspace}/ai/sessions/push: + post: + summary: write changed pieces of AI sessions to their backups, and remove deleted ones + operationId: pushAiSessionBackups + tags: + - ai + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - owner + properties: + owner: + type: string + description: the email the push was prepared for; refused with a 409 when it is not the caller's + sessions: + type: array + items: + $ref: "#/components/schemas/AISessionBackupPush" + removed: + type: array + items: + type: string + responses: + "200": + description: one result per session written or removed, in request order + content: + application/json: + schema: + type: object + required: + - enabled + - results + properties: + enabled: + type: boolean + storage_id: + type: string + backup_generation: + type: integer + results: + type: array + items: + type: object + required: + - id + properties: + id: + type: string + error: + type: string + needs_whole: + type: boolean + description: nothing was written and the session must be pushed whole again; an incremental part found no listed session to ride on (the backup was removed, or a push split over parts is in progress or was abandoned), or a later part of a push split over parts found another push had superseded it + /w/{workspace}/ai/shared_artifacts/share: post: summary: share an AI session artifact with the workspace @@ -28289,6 +28436,155 @@ components: fixes) from the workspace UI. Read from the workspace's own settings even when the providers served fall back to the instance config. AI agent steps and the AI sandbox in flows are unaffected. + sessions_storage_disabled: + type: boolean + description: >- + Stops browsers from backing their AI sessions up to the workspace's object + storage. Read from the workspace's own settings like `copilot_disabled`. + + AISessionBackupListing: + type: object + required: + - id + - updated_at + - epoch + properties: + id: + type: string + updated_at: + type: string + format: date-time + epoch: + type: integer + description: the session's move count when this copy was pushed; of a session two workspaces list, the copy with the higher one is the later + + AISessionBackupImage: + type: object + required: + - chat_id + - id + - data_url + properties: + chat_id: + type: string + id: + type: string + data_url: + type: string + + AISessionBackupChat: + type: object + required: + - id + - record + properties: + id: + type: string + record: + type: object + additionalProperties: true + + AISessionBackup: + type: object + required: + - id + - head + - chats + - images + - listing + properties: + id: + type: string + head: + type: object + additionalProperties: true + chats: + type: array + items: + $ref: "#/components/schemas/AISessionBackupChat" + images: + type: array + items: + $ref: "#/components/schemas/AISessionBackupImage" + artifacts: + type: object + additionalProperties: true + next: + $ref: "#/components/schemas/AISessionBackupCursor" + listing: + type: string + description: a fingerprint of the session's listing; pages of one session whose fingerprints differ do not belong together + moved: + type: boolean + description: the backup kept changing while this page was read, so it may mix two versions; the browser starts the session over + + AISessionBackupCursor: + type: object + description: where a pull of a session that did not fit one answer whole picks up; the rest of the session follows a pull naming that session alone with this as `resume` + required: + - id + - images + - after + properties: + id: + type: string + images: + type: boolean + after: + type: string + + AISessionBackupPush: + type: object + required: + - id + properties: + id: + type: string + head: + type: object + additionalProperties: true + chats: + type: array + items: + $ref: "#/components/schemas/AISessionBackupChat" + images: + type: array + items: + $ref: "#/components/schemas/AISessionBackupImage" + artifacts: + type: object + additionalProperties: true + delete_chats: + type: array + items: + type: string + delete_images: + type: array + items: + type: object + required: + - chat_id + - id + properties: + chat_id: + type: string + id: + type: string + partial: + type: boolean + description: more parts of this session follow, in this push or a later one; the session is not listed on this one. Such a part names its push (`push`), or it is refused + whole: + type: boolean + description: a part of a push of the session whole; the head is on the part that opens it, which replaces whatever the storage holds of the session, and every piece the browser has is on one of them. An incremental part instead rides on a session the storage lists and is refused with needs_whole when it lists none + push: + type: string + description: a push split over several parts names itself on each with a token the browser draws; the part that opens it unlists the session and the last part lists it again, and a later part is written only while that token is the one there (refused with needs_whole otherwise) + opens: + type: boolean + description: this part opens the push named by `push` + epoch: + type: integer + description: the session's move count (its record's `moves`), kept with the marker that lists the session; an incremental part rides on the marker of the same count FreeTierInfo: type: object diff --git a/backend/windmill-api/src/ai.rs b/backend/windmill-api/src/ai.rs index 8101064c3d..8b9c7929ab 100644 --- a/backend/windmill-api/src/ai.rs +++ b/backend/windmill-api/src/ai.rs @@ -451,6 +451,10 @@ pub struct AIConfig { /// and the AI sandbox are unaffected, so the providers stay in force. #[serde(default, skip_serializing_if = "std::ops::Not::not")] pub copilot_disabled: bool, + /// Stops browsers from backing their AI sessions up to the workspace's object storage + /// (`ai_sessions.rs`). Read from the workspace's own row like `copilot_disabled`. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub sessions_storage_disabled: bool, } /// Negotiated rates in USD per million tokens. An unset cache rate is read as the @@ -527,6 +531,9 @@ pub fn workspaced_service() -> Router { #[cfg(feature = "bedrock")] let router = router.route("/check_bedrock_credentials", get(check_bedrock_credentials)); + #[cfg(feature = "parquet")] + let router = router.nest("/sessions", crate::ai_sessions::workspaced_service()); + router } diff --git a/backend/windmill-api/src/ai_sessions.rs b/backend/windmill-api/src/ai_sessions.rs new file mode 100644 index 0000000000..fca9c35fca --- /dev/null +++ b/backend/windmill-api/src/ai_sessions.rs @@ -0,0 +1,1316 @@ +//! Lazily replicated backups of the browser's AI sessions in the workspace's object storage. +//! +//! The browser keeps the sessions in IndexedDB and pushes changed pieces here in batches; an +//! empty browser restores from what was pushed. The server owns the key layout, keeps the +//! caller's own prefix the only one it can reach, and encrypts every object with the +//! workspace key so bucket credentials do not read transcripts: +//! +//! ```text +//! windmill_ai_sessions/{w_id}/g{generation}/{sha256(email)}/sessions/{sid}/head.json +//! windmill_ai_sessions/{w_id}/g{generation}/{sha256(email)}/sessions/{sid}/chats/{cid}.json +//! windmill_ai_sessions/{w_id}/g{generation}/{sha256(email)}/sessions/{sid}/artifacts.json +//! windmill_ai_sessions/{w_id}/g{generation}/{sha256(email)}/images/{sid}/{cid}/{iid} +//! windmill_ai_sessions/{w_id}/g{generation}/{sha256(email)}/index/{sid}/{epoch} +//! ``` +//! +//! The index marker is empty, written last by every push of the session, and is what a +//! listing reads: one object per session, whatever the session holds, its `last_modified` +//! the session's `updated_at`. + +use crate::db::{ApiAuthed, DB}; +use axum::{ + extract::{DefaultBodyLimit, Path}, + routing::{get, post}, + Extension, Json, Router, +}; +use futures::{StreamExt, TryStreamExt}; +use magic_crypt::{MagicCrypt256, MagicCryptTrait}; +use serde::{Deserialize, Serialize}; +use serde_json::value::RawValue; +use std::sync::Arc; +use windmill_api_auth::is_effectively_unscoped; +use windmill_api_workspaces::ai_session_backups::{ + generation_prefix, storage_id, MAX_OBJECT_BYTES, +}; +use windmill_common::error::{Error, JsonResult, Result}; +use windmill_common::utils::calculate_hash; +use windmill_common::variables::{crypt_from_key_with_suffix, get_workspace_key}; +use windmill_object_store::object_store_reexports::{ + ObjectStore, ObjectStoreError, Path as ObjectPath, PutPayload, +}; +use windmill_object_store::{build_object_store_client, object_store_error_to_error}; + +const PUSH_BODY_LIMIT: usize = MAX_OBJECT_BYTES; +/// A pull names at most MAX_PULL_IDS ids of 64 bytes; anything larger is not a pull. +const PULL_BODY_LIMIT: usize = 64 * 1024; +/// A pull answer larger than this hands the remaining ids back as `deferred`. +const PULL_RESPONSE_BUDGET: usize = 32 * 1024 * 1024; +const MAX_HEAD_BYTES: usize = 1024 * 1024; +/// What the cipher adds to a plaintext at most (a block of padding): an object stored at a +/// cap is that much larger than the cap when read back. +const CIPHER_PADDING: usize = 16; +/// The browser bounds an image to a 1568 px edge and re-encodes past 700 KB; this is +/// well above what that produces. +const MAX_IMAGE_BYTES: usize = 4 * 1024 * 1024; +const MAX_PULL_IDS: usize = 20; +const MAX_PUSH_SESSIONS: usize = 100; +const MAX_REMOVED: usize = 200; +const MAX_CHATS_PER_ENTRY: usize = 100; +const MAX_IMAGES_PER_ENTRY: usize = 500; +const MAX_DELETES_PER_ENTRY: usize = 1000; +const MAX_OPERATIONS_PER_PUSH: usize = 4000; +/// Entries of listing metadata a pull holds per page of a session. +const MAX_LISTED_OBJECTS: usize = 5000; +/// Session markers a listing scans, and the newest sessions it answers with. +const MAX_LIST_SCAN: usize = 50_000; +const LIST_MAX: usize = 500; +const IO_CONCURRENCY: usize = 8; + +pub fn workspaced_service() -> Router { + Router::new() + .route("/list", get(list)) + .route( + "/pull", + post(pull).layer(DefaultBodyLimit::max(PULL_BODY_LIMIT)), + ) + .route( + "/push", + post(push).layer(DefaultBodyLimit::max(PUSH_BODY_LIMIT)), + ) +} + +/// What reading an object yields. `Gone`: not there (deleted since the listing, or never +/// pushed). `Grown`: larger than expected, so replaced since the listing (or planted), and +/// left unread; a pull answers with a page ending before it rather than without it. +/// `Foreign`: it does not decrypt for this user (written under another user's or +/// workspace's key), and must not take the rest of the session down with it. +enum Read { + Text(String), + Gone, + Grown, + Foreign, +} + +/// The user's prefix in the workspace storage, plus what reads and writes it. +struct Backend { + store: Arc, + mc: MagicCrypt256, + prefix: String, + /// Name the storage and the generation the objects are under, for the browser's sync + /// state: a row recorded against another storage or generation is stale, a removal is + /// owed to the storage alone (a rotation deleted the older generation's copy anyway). + storage_id: String, + generation: i64, +} + +impl Backend { + fn index_prefix(&self) -> ObjectPath { + ObjectPath::from(format!("{}/index/", self.prefix)) + } + + /// The marker that lists the session, named by the session's move count so that of a + /// session two workspaces list, the copy moved last is told from the listing alone. + fn index_key(&self, sid: &str, epoch: u32) -> ObjectPath { + ObjectPath::from(format!("{}/index/{sid}/{epoch}", self.prefix)) + } + + fn index_session_prefix(&self, sid: &str) -> ObjectPath { + ObjectPath::from(format!("{}/index/{sid}/", self.prefix)) + } + + /// The token of the push split over parts in progress, under the session so a removal + /// or the next whole push clears it with the rest. + fn push_key(&self, sid: &str) -> ObjectPath { + ObjectPath::from(format!("{}/sessions/{sid}/push", self.prefix)) + } + + fn session_prefix(&self, sid: &str) -> ObjectPath { + ObjectPath::from(format!("{}/sessions/{sid}/", self.prefix)) + } + + fn head_key(&self, sid: &str) -> ObjectPath { + ObjectPath::from(format!("{}/sessions/{sid}/head.json", self.prefix)) + } + + fn chat_key(&self, sid: &str, cid: &str) -> ObjectPath { + ObjectPath::from(format!("{}/sessions/{sid}/chats/{cid}.json", self.prefix)) + } + + fn artifacts_key(&self, sid: &str) -> ObjectPath { + ObjectPath::from(format!("{}/sessions/{sid}/artifacts.json", self.prefix)) + } + + fn images_prefix(&self, sid: &str) -> ObjectPath { + ObjectPath::from(format!("{}/images/{sid}/", self.prefix)) + } + + fn chat_images_prefix(&self, sid: &str, cid: &str) -> ObjectPath { + ObjectPath::from(format!("{}/images/{sid}/{cid}/", self.prefix)) + } + + fn image_key(&self, sid: &str, cid: &str, iid: &str) -> ObjectPath { + ObjectPath::from(format!("{}/images/{sid}/{cid}/{iid}", self.prefix)) + } + + fn seal(&self, plaintext: &[u8]) -> Vec { + self.mc.encrypt_bytes_to_bytes(plaintext) + } + + /// Bytes written. + async fn put_sealed(&self, key: &ObjectPath, ciphertext: Vec) -> Result { + let written = ciphertext.len(); + self.store + .put(key, PutPayload::from(ciphertext)) + .await + .map_err(object_store_error_to_error)?; + Ok(written) + } + + async fn put(&self, key: &ObjectPath, plaintext: &[u8]) -> Result { + self.put_sealed(key, self.seal(plaintext)).await + } + + /// `max` is what the listing said the object holds, or the cap of its kind for one read + /// without a listing: checked before buffering, since whoever holds the bucket's + /// credentials can put anything at a predictable key. + async fn get(&self, key: &ObjectPath, max: usize) -> Result { + let result = match self.store.get(key).await { + Ok(result) => result, + Err(ObjectStoreError::NotFound { .. }) => return Ok(Read::Gone), + Err(e) => return Err(object_store_error_to_error(e)), + }; + let size = result.meta.size as usize; + // Larger than any push writes: planted, whatever the listing said, and skipped like + // an object of another key rather than retried like one that grew. + if size > MAX_OBJECT_BYTES { + tracing::warn!("AI session backup object {key} is larger than any push writes"); + return Ok(Read::Foreign); + } + if size > max { + return Ok(Read::Grown); + } + let bytes = result.bytes().await.map_err(object_store_error_to_error)?; + // The objects are JSON and data URLs: a wrong key's output failing UTF-8 tells it + // apart beyond the cipher's padding check, which a wrong key passes now and then. + match self + .mc + .decrypt_bytes_to_bytes(&bytes) + .ok() + .and_then(|plaintext| String::from_utf8(plaintext).ok()) + { + Some(text) => Ok(Read::Text(text)), + None => { + tracing::warn!("AI session backup object {key} does not decrypt for its reader"); + Ok(Read::Foreign) + } + } + } + + async fn delete(&self, key: &ObjectPath) -> Result<()> { + match self.store.delete(key).await { + Ok(()) | Err(ObjectStoreError::NotFound { .. }) => Ok(()), + Err(e) => Err(object_store_error_to_error(e)), + } + } + + /// The entries under `prefix` past `after` in key order, as many as fit `budget` bytes; + /// `true` when more follow. Every key past `after` is seen and the MAX_LISTED_OBJECTS + /// smallest kept (a max-heap dropping its largest), since a page is defined by key + /// order and the store promises none; that cap is what bounds a pull's memory, a + /// session growing by valid pushes without limit. With `at_least_one`, the first entry + /// is taken whatever its size, so an answer owed the session makes progress on it (no + /// object exceeds the push body cap). + async fn list_within( + &self, + prefix: &ObjectPath, + after: Option<&ObjectPath>, + budget: usize, + at_least_one: bool, + ) -> Result<(Vec<(ObjectPath, usize)>, bool)> { + let mut kept: std::collections::BinaryHeap<(ObjectPath, usize)> = Default::default(); + let mut dropped = false; + let mut stream = match after { + Some(after) => self.store.list_with_offset(Some(prefix), after), + None => self.store.list(Some(prefix)), + }; + while let Some(meta) = stream.next().await { + let meta = meta.map_err(object_store_error_to_error)?; + kept.push((meta.location, meta.size as usize)); + if kept.len() > MAX_LISTED_OBJECTS { + kept.pop(); + dropped = true; + } + } + let mut entries = vec![]; + let mut total = 0; + for (key, size) in kept.into_sorted_vec() { + if total + size > budget && !(at_least_one && entries.is_empty()) { + return Ok((entries, true)); + } + total += size; + entries.push((key, size)); + } + Ok((entries, dropped)) + } + + /// Bytes written. Sealed up front so every stream item is owned: an item borrowing + /// from the request makes the future higher-ranked over that lifetime, which the + /// handler's `Send` bound cannot prove. + async fn put_all(&self, puts: Vec<(ObjectPath, Vec)>) -> Result { + futures::stream::iter(puts) + .map(|(key, ciphertext)| async move { self.put_sealed(&key, ciphertext).await }) + .buffer_unordered(IO_CONCURRENCY) + .try_fold(0, |acc, n| async move { Ok::<_, Error>(acc + n) }) + .await + } + + async fn delete_all(&self, keys: Vec) -> Result<()> { + futures::stream::iter(keys) + .map(|key| async move { self.delete(&key).await }) + .buffer_unordered(IO_CONCURRENCY) + .try_collect::>() + .await?; + Ok(()) + } + + /// A fingerprint of the session's marker and of everything listed under its two + /// prefixes (key, size, modification time, entity tag and version), combined as the + /// listing streams and in no particular order, so a session of any size costs bounded + /// memory. `None` for a session + /// the storage does not list. Taken before and after a page is read, so a page a push + /// changed under is read again; pages of one pull carry it, and the browser starts the + /// session over when it moved between two of them. + async fn listing_fingerprint(&self, sid: &str) -> Result> { + use std::hash::{DefaultHasher, Hash, Hasher}; + // The entity tag and version go in with the key, size and time: a store reports + // modification times coarsely, and an object rewritten at the same size within that + // grain would otherwise fingerprint the same. + fn fold( + acc: u64, + location: &str, + size: S, + modified: i64, + e_tag: Option<&str>, + version: Option<&str>, + ) -> u64 { + let mut hasher = DefaultHasher::new(); + (location, size, modified, e_tag, version).hash(&mut hasher); + acc.wrapping_add(hasher.finish()) + } + let mut acc = 0u64; + let mut listed = false; + for (marker, prefix) in [ + (true, self.index_session_prefix(sid)), + (false, self.session_prefix(sid)), + (false, self.images_prefix(sid)), + ] { + let mut stream = self.store.list(Some(&prefix)); + while let Some(meta) = stream.next().await { + let meta = meta.map_err(object_store_error_to_error)?; + listed |= marker; + acc = fold( + acc, + meta.location.as_ref(), + meta.size, + meta.last_modified.timestamp_millis(), + meta.e_tag.as_deref(), + meta.version.as_deref(), + ); + } + } + if !listed { + return Ok(None); + } + Ok(Some(format!("{acc:016x}"))) + } + + async fn exists(&self, key: &ObjectPath) -> Result { + match self.store.head(key).await { + Ok(_) => Ok(true), + Err(ObjectStoreError::NotFound { .. }) => Ok(false), + Err(e) => Err(object_store_error_to_error(e)), + } + } + + /// Deletes as the listing streams, so a prefix of any size costs bounded memory. + async fn delete_prefix(&self, prefix: &ObjectPath) -> Result<()> { + self.store + .list(Some(prefix)) + .map_err(object_store_error_to_error) + .try_for_each_concurrent(IO_CONCURRENCY, |meta| async move { + self.delete(&meta.location).await + }) + .await + } +} + +/// Backups are the user's own browser state and nothing else may reach them: a job token +/// may carry an `on_behalf_of` identity, and every scoped token (guest, embed, app policy, +/// MCP) is minted for something narrower than the user's whole assistant history. +fn require_plain_user_token(authed: &ApiAuthed) -> Result<()> { + if authed.job_id.is_some() || !is_effectively_unscoped(authed.scopes.as_deref()) { + return Err(Error::PermissionDenied( + "AI session backups are only reachable with an unscoped user token".to_string(), + )); + } + Ok(()) +} + +/// Every key is assembled server-side from ids the browser mints (`createLongHash` and +/// `randomUUID` forms), so anything outside this alphabet is a forged id, not a real one. +fn require_valid_id(kind: &str, id: &str) -> Result<()> { + let ok = !id.is_empty() + && id.len() <= 64 + && id + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'_' || b == b'-'); + if !ok { + return Err(Error::BadRequest(format!("invalid {kind} id: {id:?}"))); + } + Ok(()) +} + +/// Images travel as base64 data URLs and are stored verbatim, so they serialize back into +/// a pull answer at exactly their stored size; anything else (control characters, +/// quotes) could grow several times under JSON escaping and defeat the pull budget. +fn require_data_url(data_url: &str) -> Result<()> { + let ok = data_url.len() <= MAX_IMAGE_BYTES + && data_url + .strip_prefix("data:") + .and_then(|rest| rest.split_once(";base64,")) + .is_some_and(|(mime, payload)| { + !mime.is_empty() + && mime.bytes().all(|b| { + b.is_ascii_alphanumeric() || matches!(b, b'/' | b'.' | b'+' | b'-') + }) + && payload + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'+' | b'/' | b'=')) + }); + if !ok { + return Err(Error::BadRequest( + "an image must be a base64 data URL within the size cap".to_string(), + )); + } + Ok(()) +} + +fn require_json_object(kind: &str, raw: &RawValue, max_bytes: usize) -> Result<()> { + let text = raw.get(); + if !text.trim_start().starts_with('{') { + return Err(Error::BadRequest(format!("{kind} must be a JSON object"))); + } + if text.len() > max_bytes { + return Err(Error::BadRequest(format!( + "{kind} exceeds {max_bytes} bytes" + ))); + } + Ok(()) +} + +/// `None` when the workspace has nowhere to keep backups: no primary storage configured, or +/// the admin switched them off. Both read as `enabled: false` so the browser stops trying. +async fn backend(authed: &ApiAuthed, db: &DB, w_id: &str) -> Result> { + let (disabled, generation) = sqlx::query_as::<_, (Option, i64)>( + "SELECT (ai_config->>'sessions_storage_disabled')::bool, ai_sessions_backup_generation \ + FROM workspace_settings WHERE workspace_id = $1", + ) + .bind(w_id) + .fetch_optional(db) + .await? + .unwrap_or((None, 0)); + if disabled.unwrap_or(false) { + return Ok(None); + } + let (_, resource) = + crate::job_helpers_oss::get_workspace_s3_resource(authed, db, None, w_id, None).await?; + let Some(resource) = resource else { + return Ok(None); + }; + let store = build_object_store_client(&resource).await?; + let user = calculate_hash(&authed.email); + // Keyed per user, not per workspace: anyone who can write the bucket could otherwise copy + // another member's ciphertext under their own prefix and have `pull` decrypt it for them. + let key = get_workspace_key(w_id, db).await?; + let mc = crypt_from_key_with_suffix(&key, &user); + let storage_id = storage_id(&resource); + let prefix = format!("{}/{user}", generation_prefix(w_id, generation)); + Ok(Some(Backend { store, mc, prefix, storage_id, generation })) +} + +#[derive(Serialize)] +struct SessionListing { + id: String, + updated_at: chrono::DateTime, + /// The session's move count when this copy was pushed (see `PushedSession::epoch`). + epoch: u32, +} + +#[derive(Serialize)] +struct ListResponse { + enabled: bool, + /// The storage answered from, and the generation a key rotation bumps; a browser whose + /// sync state names another storage or generation starts over. + #[serde(skip_serializing_if = "Option::is_none")] + storage_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + backup_generation: Option, + sessions: Vec, + /// The user has more sessions than the answer names. + #[serde(skip_serializing_if = "std::ops::Not::not")] + truncated: bool, +} + +/// A session is listed once a push entry of it landed whole (its marker is written last); +/// a push that failed before that left objects the listing does not name. +async fn list( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, +) -> JsonResult { + require_plain_user_token(&authed)?; + let Some(backend) = backend(&authed, &db, &w_id).await? else { + return Ok(Json(ListResponse { + enabled: false, + storage_id: None, + backup_generation: None, + sessions: vec![], + truncated: false, + })); + }; + let prefix = backend.index_prefix(); + let mut stream = backend.store.list(Some(&prefix)); + // One marker per session, whatever the session holds: the newest LIST_MAX are kept as + // the scan goes (a min-heap drops the oldest), and the scan itself is bounded. + let mut newest: std::collections::BinaryHeap< + std::cmp::Reverse<(chrono::DateTime, u32, String)>, + > = Default::default(); + let mut scanned = 0; + let mut truncated = false; + while let Some(meta) = stream.next().await { + let meta = meta.map_err(object_store_error_to_error)?; + scanned += 1; + if scanned > MAX_LIST_SCAN { + truncated = true; + break; + } + // `Path` drops the trailing delimiter, so the remainder starts with one. + let Some(rel) = meta.location.as_ref().strip_prefix(prefix.as_ref()) else { + continue; + }; + let Some((sid, epoch)) = rel.trim_start_matches('/').split_once('/') else { + continue; + }; + let Ok(epoch) = epoch.parse::() else { + continue; + }; + if sid.is_empty() || sid.contains('/') { + continue; + } + newest.push(std::cmp::Reverse(( + meta.last_modified, + epoch, + sid.to_string(), + ))); + if newest.len() > LIST_MAX { + newest.pop(); + truncated = true; + } + } + let mut sessions: Vec = newest + .into_iter() + .map(|std::cmp::Reverse((updated_at, epoch, id))| SessionListing { id, updated_at, epoch }) + .collect(); + sessions.sort_by(|a, b| b.updated_at.cmp(&a.updated_at)); + Ok(Json(ListResponse { + enabled: true, + storage_id: Some(backend.storage_id.clone()), + backup_generation: Some(backend.generation), + sessions, + truncated, + })) +} + +#[derive(Deserialize)] +struct PullRequest { + ids: Vec, + /// Picks the session an earlier answer cut up from where it stopped; `ids` then names + /// that session alone. + #[serde(default)] + resume: Option, +} + +/// Where a pull of a session that outgrew one answer picks up: the last key the earlier +/// answer carried, in the session's prefix or, once that one is done, in its images prefix. +#[derive(Serialize, Deserialize, Clone)] +struct PullCursor { + id: String, + images: bool, + after: String, +} + +#[derive(Serialize)] +struct PulledChat { + id: String, + record: Box, +} + +#[derive(Serialize, Deserialize)] +struct ImageObject { + chat_id: String, + id: String, + data_url: String, +} + +#[derive(Serialize)] +struct PulledSession { + id: String, + head: Box, + chats: Vec, + images: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + artifacts: Option>, + /// The session did not fit this answer whole: the rest follows a pull with this cursor. + #[serde(skip_serializing_if = "Option::is_none")] + next: Option, + /// A fingerprint of the session's listing (marker, and every key, size, modification + /// time, entity tag and version), so the browser tells that the backup changed between + /// the pages it assembled. + listing: String, + /// The backup kept changing while this page was read (a push landing object by object), + /// so the page may mix two versions: the browser starts the session over. + #[serde(skip_serializing_if = "std::ops::Not::not")] + moved: bool, +} + +/// How many times a page whose listing moved while it was read is read again before it is +/// handed over as `moved`. +const PULL_REREADS: usize = 3; + +#[derive(Serialize)] +struct PullResponse { + enabled: bool, + #[serde(skip_serializing_if = "Option::is_none")] + storage_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + backup_generation: Option, + sessions: Vec, + /// Ids that did not fit the response budget; ask for them again. + deferred: Vec, +} + +fn raw(kind: &str, text: String) -> Result> { + RawValue::from_string(text) + .map_err(|e| Error::internal_err(format!("stored {kind} is not JSON: {e}"))) +} + +enum PullStep { + Absent, + Deferred, + Fetched(PulledSession, usize), +} + +/// Fetch one session: its head, then every chat and artifact object under its prefix, and +/// as many of its images as the budget allows (a missing image hydrates to a placeholder +/// in the browser). Sizes come from the listings, so a session that would not fit is +/// deferred before anything of it is read, unless it is the first of the response, which +/// must carry something. `Absent` when it has no head. +async fn pull_session( + backend: &Backend, + sid: &str, + budget: usize, + first: bool, + resume: Option<&PullCursor>, +) -> Result { + // A page read while a push lands object by object may mix two versions of the session: + // the listing is taken again once the page is read, and a page it moved under is read + // again, a few times, then handed over as such for the browser to start over. + for reread in 0..PULL_REREADS { + let step = pull_page(backend, sid, budget, first, resume).await?; + let PullStep::Fetched(mut page, size) = step else { + return Ok(step); + }; + if backend.listing_fingerprint(sid).await?.as_deref() == Some(page.listing.as_str()) { + return Ok(PullStep::Fetched(page, size)); + } + if reread + 1 == PULL_REREADS { + page.moved = true; + return Ok(PullStep::Fetched(page, size)); + } + } + unreachable!("a page is answered on the last reread") +} + +async fn pull_page( + backend: &Backend, + sid: &str, + budget: usize, + first: bool, + resume: Option<&PullCursor>, +) -> Result { + // Taken before anything of the page is listed or read: an object landing after it is + // in the next page's fingerprint, whereas one landing after the reads but before a + // fingerprint taken then would have certified a page without it. A session the storage + // does not list (removed, or a whole push in progress) is absent. + let Some(listing) = backend.listing_fingerprint(sid).await? else { + return Ok(PullStep::Absent); + }; + let Read::Text(head) = backend + .get(&backend.head_key(sid), MAX_HEAD_BYTES + CIPHER_PADDING) + .await? + else { + return Ok(PullStep::Absent); + }; + let session_prefix = backend.session_prefix(sid); + let images_prefix = backend.images_prefix(sid); + let mut size = head.len(); + let mut chats = vec![]; + let mut artifacts = None; + let mut images = vec![]; + let mut next = None; + let cursor = |images: bool, after: String| PullCursor { id: sid.to_string(), images, after }; + // Sizes come from the listings, and the listings stop at the budget, so nothing is read + // past it even for the first session of the answer. One that outgrew it (chats + // accumulate over pushes) comes back in pages, in key order, each answer naming where + // the next picks up; the browser imports nothing before the last page. An object that + // grew since the listing (a push replaced it) ends the page just before it, and the + // answer names that spot: a new listing sizes it, whereas dropping it would import the + // session without it for good. + let in_images = resume.is_some_and(|c| c.images); + if !in_images { + let after = resume.map(|c| ObjectPath::from(c.after.as_str())); + let (entries, cut) = backend + .list_within( + &session_prefix, + after.as_ref(), + budget.saturating_sub(size), + first, + ) + .await?; + if cut && !first { + return Ok(PullStep::Deferred); + } + if cut { + next = entries + .last() + .map(|(key, _)| cursor(false, key.to_string())); + } + let to_read: Vec<(ObjectPath, usize, Option)> = entries + .into_iter() + .filter_map(|(key, bytes)| { + let rel = key + .as_ref() + .strip_prefix(session_prefix.as_ref()) + .unwrap_or_default() + .trim_start_matches('/'); + if rel == "artifacts.json" { + Some((key, bytes, None)) + } else { + let cid = rel + .strip_prefix("chats/")? + .strip_suffix(".json")? + .to_string(); + Some((key, bytes, Some(cid))) + } + }) + .collect(); + let reads: Vec<(ObjectPath, usize, Option, Read)> = futures::stream::iter(to_read) + .map(|(key, bytes, cid)| async move { + let read = backend.get(&key, bytes).await?; + Ok::<_, Error>((key, bytes, cid, read)) + }) + .buffered(IO_CONCURRENCY) + .try_collect() + .await?; + let mut before = resume.map(|c| c.after.clone()).unwrap_or_default(); + for (key, bytes, cid, read) in reads { + match (cid, read) { + (_, Read::Grown) => { + next = Some(cursor(false, before)); + break; + } + (None, Read::Text(text)) => { + artifacts = Some(raw("artifacts", text)?); + size += bytes; + } + (Some(cid), Read::Text(text)) => { + chats.push(PulledChat { id: cid, record: raw("chat", text)? }); + size += bytes; + } + _ => {} + } + before = key.to_string(); + } + } + if next.is_none() { + let after = resume + .filter(|c| c.images) + .map(|c| ObjectPath::from(c.after.as_str())); + let (entries, cut) = backend + .list_within( + &images_prefix, + after.as_ref(), + budget.saturating_sub(size), + first, + ) + .await?; + let mut before = after.map(|a| a.to_string()).unwrap_or_default(); + if cut { + // An answer with no room for a single image names where it stood, so the pull + // owed the session alone picks it up there. + next = Some(cursor( + true, + entries + .last() + .map(|(key, _)| key.to_string()) + .unwrap_or_else(|| before.clone()), + )); + } + let to_read: Vec<(ObjectPath, usize, String, String)> = entries + .into_iter() + .filter_map(|(key, bytes)| { + let rel = key.as_ref().strip_prefix(images_prefix.as_ref())?; + let (cid, iid) = rel.trim_start_matches('/').split_once('/')?; + Some((key.clone(), bytes, cid.to_string(), iid.to_string())) + }) + .collect(); + let reads: Vec<(ObjectPath, usize, String, String, Read)> = futures::stream::iter(to_read) + .map(|(key, bytes, cid, iid)| async move { + let read = backend.get(&key, bytes).await?; + Ok::<_, Error>((key, bytes, cid, iid, read)) + }) + .buffered(IO_CONCURRENCY) + .try_collect() + .await?; + for (key, bytes, chat_id, id, read) in reads { + match read { + Read::Grown => { + next = Some(cursor(true, before)); + break; + } + Read::Text(data_url) => { + images.push(ImageObject { chat_id, id, data_url }); + size += bytes; + } + _ => {} + } + before = key.to_string(); + } + } + Ok(PullStep::Fetched( + PulledSession { + id: sid.to_string(), + head: raw("head", head)?, + chats, + images, + artifacts, + next, + listing, + moved: false, + }, + size, + )) +} + +async fn pull( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Json(req): Json, +) -> JsonResult { + require_plain_user_token(&authed)?; + if req.ids.len() > MAX_PULL_IDS { + return Err(Error::BadRequest(format!( + "at most {MAX_PULL_IDS} sessions per pull" + ))); + } + for id in &req.ids { + require_valid_id("session", id)?; + } + if let Some(cursor) = &req.resume { + if req.ids.len() != 1 || req.ids[0] != cursor.id || cursor.after.len() > 1024 { + return Err(Error::BadRequest( + "a resumed pull names the resumed session alone".to_string(), + )); + } + } + let Some(backend) = backend(&authed, &db, &w_id).await? else { + return Ok(Json(PullResponse { + enabled: false, + storage_id: None, + backup_generation: None, + sessions: vec![], + deferred: vec![], + })); + }; + let mut sessions = vec![]; + let mut deferred = vec![]; + let mut budget = PULL_RESPONSE_BUDGET; + for sid in req.ids { + let resume = req.resume.as_ref().filter(|c| c.id == sid); + match pull_session(&backend, &sid, budget, sessions.is_empty(), resume).await? { + PullStep::Absent => {} + PullStep::Deferred => deferred.push(sid), + PullStep::Fetched(session, size) => { + budget = budget.saturating_sub(size); + sessions.push(session); + } + } + } + Ok(Json(PullResponse { + enabled: true, + storage_id: Some(backend.storage_id), + backup_generation: Some(backend.generation), + sessions, + deferred, + })) +} + +#[derive(Deserialize)] +struct PushedChat { + id: String, + record: Box, +} + +#[derive(Deserialize)] +struct ImageRef { + chat_id: String, + id: String, +} + +#[derive(Deserialize)] +struct PushedSession { + id: String, + #[serde(default)] + head: Option>, + #[serde(default)] + chats: Vec, + #[serde(default)] + images: Vec, + #[serde(default)] + artifacts: Option>, + #[serde(default)] + delete_chats: Vec, + #[serde(default)] + delete_images: Vec, + /// More parts of the session follow, in this push or a later one: the session is not + /// listed on this one. + #[serde(default)] + partial: bool, + /// A part of a push of the session whole: the head is on the part that opens it, which + /// replaces whatever the storage holds of the session, and every piece the browser has + /// is on one of them. An incremental part instead rides on a session the storage lists, + /// and is refused with `needs_whole` when it lists none. + #[serde(default)] + whole: bool, + /// A push split over several parts names itself on each of them with a token the + /// browser draws; the part that `opens` it unlists the session (a pull between two parts + /// would otherwise take a mix of old and new pieces for the backup) and the last part + /// lists it again. A later part is written only while that token is the one there, so a + /// part of a push another one superseded is refused with `needs_whole`. + #[serde(default)] + push: Option, + #[serde(default)] + opens: bool, + /// The session's move count (its record's `moves`), the marker that lists the session + /// is named by: a session moved to another workspace is listed by both until the old + /// copy's removal lands, and the copy with the higher count is the later one. An + /// incremental part rides on the marker of the same count. + #[serde(default)] + epoch: u32, +} + +#[derive(Deserialize)] +struct PushRequest { + /// The email the browser believes it is acting for. An in-place account switch can + /// leave a flush prepared for the previous user; the server refuses it rather than + /// filing that user's sessions under the caller's prefix. + owner: String, + #[serde(default)] + sessions: Vec, + #[serde(default)] + removed: Vec, +} + +#[derive(Serialize)] +struct PushResult { + id: String, + #[serde(skip_serializing_if = "Option::is_none")] + error: Option, + /// Nothing was written; the session must be pushed whole again: an incremental part + /// found no listed session to ride on (another device removed the backup, or a push + /// split over parts is in progress or was abandoned), or a later part of a push split + /// over parts found another push had superseded it. + #[serde(skip_serializing_if = "std::ops::Not::not")] + needs_whole: bool, +} + +#[derive(Serialize)] +struct PushResponse { + enabled: bool, + #[serde(skip_serializing_if = "Option::is_none")] + storage_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + backup_generation: Option, + results: Vec, +} + +fn validate_push(req: &PushRequest) -> Result<()> { + if req.sessions.len() > MAX_PUSH_SESSIONS || req.removed.len() > MAX_REMOVED { + return Err(Error::BadRequest( + "too many sessions in one push".to_string(), + )); + } + // Every nested entry costs an object-store call (a deleted chat two), so the lists are + // bounded per entry and across the request; the browser sends far fewer. + let mut operations = req.removed.len(); + for s in &req.sessions { + if s.chats.len() > MAX_CHATS_PER_ENTRY + || s.images.len() > MAX_IMAGES_PER_ENTRY + || s.delete_chats.len() > MAX_DELETES_PER_ENTRY + || s.delete_images.len() > MAX_DELETES_PER_ENTRY + { + return Err(Error::BadRequest(format!( + "too many pieces for session {} in one push", + s.id + ))); + } + operations += s.chats.len() + s.images.len() + s.delete_chats.len() + s.delete_images.len(); + } + if operations > MAX_OPERATIONS_PER_PUSH { + return Err(Error::BadRequest("too many pieces in one push".to_string())); + } + for sid in &req.removed { + require_valid_id("session", sid)?; + } + for s in &req.sessions { + require_valid_id("session", &s.id)?; + if let Some(token) = &s.push { + require_valid_id("push", token)?; + } else if s.opens || s.partial { + // A part more parts follow belongs to a push split over parts, which names + // itself: without the token the session would stay listed between the parts. + return Err(Error::BadRequest(format!( + "session {} is pushed in parts with no push token", + s.id + ))); + } + if s.whole && (s.push.is_none() || s.opens) && s.head.is_none() { + return Err(Error::BadRequest(format!( + "session {} is pushed whole without its head", + s.id + ))); + } + if let Some(head) = &s.head { + require_json_object("head", head, MAX_HEAD_BYTES)?; + } + for c in &s.chats { + require_valid_id("chat", &c.id)?; + require_json_object("chat record", &c.record, PUSH_BODY_LIMIT)?; + } + if let Some(a) = &s.artifacts { + require_json_object("artifacts", a, PUSH_BODY_LIMIT)?; + } + for i in &s.images { + require_valid_id("chat", &i.chat_id)?; + require_valid_id("image", &i.id)?; + require_data_url(&i.data_url)?; + } + for c in &s.delete_chats { + require_valid_id("chat", c)?; + } + for i in &s.delete_images { + require_valid_id("chat", &i.chat_id)?; + require_valid_id("image", &i.id)?; + } + } + Ok(()) +} + +fn push_payload_bytes(req: &PushRequest) -> usize { + req.sessions + .iter() + .map(|s| { + s.head.as_ref().map_or(0, |h| h.get().len()) + + s.artifacts.as_ref().map_or(0, |a| a.get().len()) + + s.chats.iter().map(|c| c.record.get().len()).sum::() + + s.images.iter().map(|i| i.data_url.len()).sum::() + }) + .sum() +} + +/// Runs under the session's lock (see `lock_session`). The part that opens a whole push +/// (the one with the head) replaces the backup: the marker goes first, so nothing lists the +/// session until the last part, then everything else. An incremental part assumes the rest +/// of the session is in the storage, which a removal since would have taken, or a push +/// split over parts may still be bringing: it is refused unless the session is listed, and +/// unlists the session itself while it changes more than one object (a pull between two +/// writes would otherwise take a mix of old and new pieces for the backup). A push split +/// over parts names itself with a token: the part that opens it unlists the session and +/// writes the token, and a later part is written only while that token is +/// the one there, so two devices pushing the session at once cannot list a mix of their +/// pieces: the push that opened later wins, the other is refused and goes again. Every +/// refusal comes before anything of the part lands. Deletes run last, and the marker only +/// by the last part, so a push cut short never leaves a listed session pointing at chats +/// that are not there. +/// Bytes written, and whether the part was refused for the session to go whole. +async fn push_session(backend: &Backend, s: &PushedSession) -> Result<(usize, bool)> { + match &s.push { + Some(token) if !s.opens => { + match backend + .get(&backend.push_key(&s.id), token.len() + CIPHER_PADDING) + .await? + { + Read::Text(current) if current == *token => {} + _ => return Ok((0, true)), + } + } + _ => { + if s.whole { + backend + .delete_prefix(&backend.index_session_prefix(&s.id)) + .await?; + backend + .delete_prefix(&backend.session_prefix(&s.id)) + .await?; + backend.delete_prefix(&backend.images_prefix(&s.id)).await?; + } else { + if !backend.exists(&backend.index_key(&s.id, s.epoch)).await? { + return Ok((0, true)); + } + // Unlisted while more than one object changes (a push split over parts, or + // one part touching several pieces): a pull between two of the writes, or + // after one of them failed, would otherwise take a mix of old and new + // pieces for the backup. One object changing is one write. + let pieces = s.chats.len() + + s.images.len() + + usize::from(s.artifacts.is_some()) + + usize::from(s.head.is_some()) + + s.delete_chats.len() + + s.delete_images.len(); + if s.push.is_some() || pieces > 1 { + backend + .delete_prefix(&backend.index_session_prefix(&s.id)) + .await?; + } + } + if let Some(token) = &s.push { + backend + .put(&backend.push_key(&s.id), token.as_bytes()) + .await?; + } + } + } + let mut written = 0; + written += backend + .put_all( + s.images + .iter() + .map(|img| { + ( + backend.image_key(&s.id, &img.chat_id, &img.id), + backend.seal(img.data_url.as_bytes()), + ) + }) + .collect(), + ) + .await?; + written += backend + .put_all( + s.chats + .iter() + .map(|c| { + ( + backend.chat_key(&s.id, &c.id), + backend.seal(c.record.get().as_bytes()), + ) + }) + .collect(), + ) + .await?; + if let Some(a) = &s.artifacts { + written += backend + .put(&backend.artifacts_key(&s.id), a.get().as_bytes()) + .await?; + } + if let Some(h) = &s.head { + written += backend + .put(&backend.head_key(&s.id), h.get().as_bytes()) + .await?; + } + for cid in &s.delete_chats { + backend.delete(&backend.chat_key(&s.id, cid)).await?; + backend + .delete_prefix(&backend.chat_images_prefix(&s.id, cid)) + .await?; + } + backend + .delete_all( + s.delete_images + .iter() + .map(|i| backend.image_key(&s.id, &i.chat_id, &i.id)) + .collect(), + ) + .await?; + if s.partial { + return Ok((written, false)); + } + // Last, and by the last part only, so a session is listed once its whole entry landed. + backend + .store + .put(&backend.index_key(&s.id, s.epoch), PutPayload::new()) + .await + .map_err(object_store_error_to_error)?; + if s.push.is_some() { + backend.delete(&backend.push_key(&s.id)).await?; + } + Ok((written, false)) +} + +/// The marker goes first so a removal cut short leaves nothing listed, then the head so +/// nothing pulls either, and no push takes it for a session still there (see `push_session`). +async fn remove_session(backend: &Backend, sid: &str) -> Result<()> { + backend + .delete_prefix(&backend.index_session_prefix(sid)) + .await?; + backend.delete(&backend.head_key(sid)).await?; + backend.delete_prefix(&backend.session_prefix(sid)).await?; + backend.delete_prefix(&backend.images_prefix(sid)).await +} + +/// One writer per session at a time, across servers: a push and a removal of the same +/// session interleaving object by object could leave a listed session missing pieces, or a +/// marker over nothing. The lock lives in a transaction that writes no rows; it is released +/// when the transaction ends. A wait past the timeout fails that entry only, and the +/// browser retries it with backoff. +async fn lock_session( + db: &DB, + backend: &Backend, + sid: &str, +) -> Result> { + let mut tx = db.begin().await?; + sqlx::query("SET LOCAL lock_timeout = '30s'") + .execute(&mut *tx) + .await?; + sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended($1, 0::int8))") + .bind(format!("ai_session_backup:{}/{sid}", backend.prefix)) + .execute(&mut *tx) + .await + .map_err(|e| { + Error::internal_err(format!( + "another device is writing the backup of session {sid}; retried later ({e})" + )) + })?; + Ok(tx) +} + +async fn push_session_locked( + db: &DB, + backend: &Backend, + s: &PushedSession, +) -> Result<(usize, bool)> { + let tx = lock_session(db, backend, &s.id).await?; + let result = push_session(backend, s).await; + tx.commit().await?; + result +} + +async fn remove_session_locked(db: &DB, backend: &Backend, sid: &str) -> Result<()> { + let tx = lock_session(db, backend, sid).await?; + let result = remove_session(backend, sid).await; + tx.commit().await?; + result +} + +async fn push( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Json(req): Json, +) -> JsonResult { + require_plain_user_token(&authed)?; + if req.owner != authed.email { + return Err(Error::Generic( + http::StatusCode::CONFLICT, + "this push was prepared for another user".to_string(), + )); + } + validate_push(&req)?; + let Some(backend) = backend(&authed, &db, &w_id).await? else { + return Ok(Json(PushResponse { + enabled: false, + storage_id: None, + backup_generation: None, + results: vec![], + })); + }; + #[cfg(not(feature = "enterprise"))] + { + let remaining = + crate::job_helpers_oss::ce_storage_quota_remaining(&db, &w_id, None).await?; + if push_payload_bytes(&req) as i64 > remaining { + return Err(Error::QuotaExceeded( + "the workspace storage quota leaves no room for this AI session backup".to_string(), + )); + } + } + #[cfg(feature = "enterprise")] + let _ = push_payload_bytes(&req); + + let mut results = Vec::with_capacity(req.sessions.len() + req.removed.len()); + let mut written: usize = 0; + // A session split into several entries is listed by the last: once one part failed, the + // later ones are not written, or the marker would list a session missing a part. + let mut failed: std::collections::HashSet<&str> = Default::default(); + for s in &req.sessions { + let (error, needs_whole) = if failed.contains(s.id.as_str()) { + ( + Some("an earlier part of this session in the push failed".to_string()), + false, + ) + } else { + match push_session_locked(&db, &backend, s).await { + Ok((n, needs_whole)) => { + written += n; + (None, needs_whole) + } + Err(e) => { + tracing::warn!("AI session backup push failed for {} in {w_id}: {e}", s.id); + failed.insert(&s.id); + (Some(e.to_string()), false) + } + } + }; + results.push(PushResult { id: s.id.clone(), error, needs_whole }); + } + for sid in &req.removed { + let error = remove_session_locked(&db, &backend, sid) + .await + .err() + .map(|e| { + tracing::warn!("AI session backup removal failed for {sid} in {w_id}: {e}"); + e.to_string() + }); + results.push(PushResult { id: sid.clone(), error, needs_whole: false }); + } + // Overwrites and deletes make this an over-count; the periodic recount the quota check + // schedules once usage is stale settles it. + #[cfg(not(feature = "enterprise"))] + if written > 0 { + crate::job_helpers_oss::bump_storage_usage( + &db, + &w_id, + windmill_object_store::DEFAULT_STORAGE, + written as i64, + ) + .await; + } + #[cfg(feature = "enterprise")] + let _ = written; + Ok(Json(PushResponse { + enabled: true, + storage_id: Some(backend.storage_id), + backup_generation: Some(backend.generation), + results, + })) +} diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index fc90852ae6..715f0b6556 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -69,6 +69,8 @@ mod ai; #[cfg(feature = "private")] mod ai_free_tier_ee; mod ai_free_tier_oss; +#[cfg(feature = "parquet")] +mod ai_sessions; mod ai_shared_artifacts; mod apps; mod apps_raw_bundle; diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index da8532e9d1..55f244c143 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -147,6 +147,7 @@ async fn edit_copilot_config( let workspace_has_config = ai_config.has_providers(); let copilot_disabled = ai_config.copilot_disabled; + let sessions_storage_disabled = ai_config.sessions_storage_disabled; let instance_ai_config = sqlx::query_scalar!("SELECT value FROM global_settings WHERE name = 'ai_config'") .fetch_optional(&db) @@ -174,6 +175,7 @@ async fn edit_copilot_config( AIConfig::default() }; effective_ai_config.copilot_disabled = copilot_disabled; + effective_ai_config.sessions_storage_disabled = sessions_storage_disabled; Ok(Json(EditCopilotConfigResponse { effective_ai_config, @@ -212,6 +214,9 @@ async fn get_copilot_info( let copilot_disabled = workspace_ai_config .as_ref() .is_some_and(|c| c.0.copilot_disabled); + let sessions_storage_disabled = workspace_ai_config + .as_ref() + .is_some_and(|c| c.0.sessions_storage_disabled); let instance_config = sqlx::query_scalar!("SELECT value FROM global_settings WHERE name = 'ai_config'") .fetch_optional(&db) @@ -236,6 +241,7 @@ async fn get_copilot_info( AIConfig::default() }; effective.copilot_disabled = copilot_disabled; + effective.sessions_storage_disabled = sessions_storage_disabled; Ok(Json(effective)) } diff --git a/backend/windmill-common/src/variables.rs b/backend/windmill-common/src/variables.rs index 045ddcd95b..1feb4182ab 100644 --- a/backend/windmill-common/src/variables.rs +++ b/backend/windmill-common/src/variables.rs @@ -252,12 +252,17 @@ pub async fn build_crypt_with_key_suffix( key_suffix: &str, ) -> crate::error::Result { let key = get_workspace_key(w_id, db).await?; + Ok(crypt_from_key_with_suffix(&key, key_suffix)) +} + +/// The cipher `build_crypt_with_key_suffix` builds, from a key string in hand. +pub fn crypt_from_key_with_suffix(key: &str, key_suffix: &str) -> MagicCrypt256 { let crypt_key = if let Some(ref salt) = SECRET_SALT.as_ref() { format!("{}{}{}", key, salt, key_suffix) } else { format!("{}{}", key, key_suffix) }; - Ok(magic_crypt::new_magic_crypt!(crypt_key, 256)) + magic_crypt::new_magic_crypt!(crypt_key, 256) } pub async fn get_workspace_key(w_id: &str, db: &DB) -> crate::error::Result { diff --git a/backend/windmill-object-store/src/lib.rs b/backend/windmill-object-store/src/lib.rs index c403625e00..25dc6ea19c 100644 --- a/backend/windmill-object-store/src/lib.rs +++ b/backend/windmill-object-store/src/lib.rs @@ -68,8 +68,8 @@ pub mod object_store_reexports { pub use object_store::path::Path; pub use object_store::{ Attribute, Attributes, Error as ObjectStoreError, GetOptions, GetRange, GetResult, - ObjectStore, PutMultipartOpts, PutPayload, PutResult, Result as ObjectStoreResult, - WriteMultipart, + ObjectMeta, ObjectStore, PutMode, PutMultipartOpts, PutOptions, PutPayload, PutResult, + Result as ObjectStoreResult, UpdateVersion, WriteMultipart, }; } diff --git a/docs/ai-session-backups.md b/docs/ai-session-backups.md new file mode 100644 index 0000000000..0b4c550456 --- /dev/null +++ b/docs/ai-session-backups.md @@ -0,0 +1,266 @@ +# AI session backups + +AI sessions live in the browser: the session list (`windmill-sessions`), chat transcripts and +image blobs (`copilot-chat-history`) and artifacts (`copilot-artifacts`), all per-user IndexedDB +stores. This is the design of their backup in the workspace's object storage, and the +constraints future work on either side must keep. + +Backend: `backend/windmill-api/src/ai_sessions.rs` (`/w/{w}/ai/sessions/{list,pull,push}`). +Frontend: `frontend/src/lib/components/sessions/sessionMirror*.ts`. + +## Why it is lazy + +A session changes at the local write rate: a transcript write every 2 s while streaming, a +session-record write per new message on screen. An object in S3 is replaced whole and every PUT +is billed, so the backup deliberately does not follow that rate. Local writes only mark a session +dirty (`sessionMirrorSignal.ts`, import-free so the stores never depend on the backup). A flush +runs 15 s after the marks go quiet, at most 2 min after the first unflushed mark, when the tab is +hidden, and 10 s after load for marks a crash left behind. Marks are persisted in localStorage +(shared by the user's tabs) for that reason, one key per mark: a shared blob would let two tabs +marking different sessions at once rewrite each other's mark away. A dirty mark is a counter +bumped on every write; a push retires it by recording the counter it covered on the session's +sync row rather than deleting the mark, since two localStorage calls cannot compare-and-delete +and a bump landing between them would be lost; retired marks are not reclaimed (one small key per +session ever backed up), and the marks of unsent drafts and of workspaces that are off stay too, +each costing one lookup per flush. Only a session gone from the store has its mark deleted. Losing the last +seconds of a device that never comes back is accepted; a tab that closes normally keeps its marks. +A signal names the user whose store the write landed in (read off the store's scoped name), so +a write that completes after the logged-in user changed marks that user's session, for their +next load, rather than the current user's. + +A flush plans and sends one session at a time, filling requests of about 8 MB as it goes, so a +first backfill of a large history never holds more than one request's worth of records and +images in memory. + +## What a push carries + +The pure planner (`sessionMirrorPlan.ts`) compares each piece against the marker of what was +last pushed, kept per session in the `windmill-sessions-mirror` store: + +| Piece | Object | Sent when | +|---|---|---| +| session record | `sessions/{sid}/head.json` | its signature changed | +| chat | `sessions/{sid}/chats/{cid}.json` | its `lastModified` moved | +| artifacts | `sessions/{sid}/artifacts.json` | their fingerprint changed | +| image | `images/{sid}/{cid}/{iid}` | never pushed before (write-once) | +| index marker | `index/{sid}/{epoch}` | last, by the part that completes a push of the session (empty; named by the record's move count) | + +All under `windmill_ai_sessions/{w_id}/g{generation}/{sha256(email)}/` in the workspace's +primary storage (the generation is what a key rotation moves, see below). +The listing reads only `index/`: one object per session whatever the session holds, so a +session with many chats cannot crowd newer ones out of a bounded scan, and its +`last_modified` is the session's `updated_at`. Written last, and only by an entry no unsent +part follows (a session split over several entries says `partial` on all but the last, and +names the push on each, or the part is refused), it +lists a session only once a whole push landed; the parts of a session after a failed one are +not written either, on the server within one push and on the client across pushes, so the +marker on the last part never lists a session missing a chat, and a new session whose last part +never lands is not listed at all. A push of the session whole (no sync row, or a stale one, +`whole` on every part) opens with the head on the first: that part replaces the backup (the +marker goes first, then everything under the session), so what an old storage still held of +the session and the push does not carry is gone. An incremental part rides on a listed +session, and the server refuses it with `needs_whole`, writing nothing, when none is listed +(a removal deletes the marker first), rather than write a marker over a session missing what +earlier parts or earlier pushes carried; one that changes more than one object unlists the +session before its writes and lists it again after them, so a pull between two of the writes, +or after one failed, finds it absent rather than a mix of old and new pieces (one object +changing is one write, and stays listed). A push split over several parts, whole or +incremental, names itself on each with a token the browser draws (`push`, `opens` on the +first): the opening part unlists the session, so a pull between two parts finds it absent +rather than a mix of old and new pieces, the last part lists it again, and a later part is +written only while that token is the one there, so two devices pushing the session at once +cannot list a mix of their pieces (the push that opened later wins; the other is refused with +`needs_whole` and goes again), and one abandoned leaves the session unlisted, so the next +push of it goes whole. A push and a removal of one session +are serialized on the server by a Postgres advisory lock keyed on the session's prefix, so +the two never interleave object by object. + +The head signature leaves out `name` (a per-browser counter the sessions page routes by), +the unsent-draft fields, `workspace_root_id` (recomputed on import), and the two fields reading +a session bumps (`lastSeenCount`, `lastActivityAt`). Reading a session must never cost a push; +keep that property when adding fields to `Session`. + +Unsent drafts (no `workspace_id`) and attached files (Blobs, directory handles) are not backed up. + +## Encryption and access + +Every object is encrypted with a key derived from the workspace key and the user +(`build_crypt_with_key_suffix` with the email hash), because workspace storage credentials are +shared far more widely than a user's transcripts: `public_resource` storages and legacy-mode +READ/WRITE hand any member the bucket. The key is per user rather than per workspace so that a +member who copies another user's ciphertext under their own prefix gets nothing from `pull`; an +object that does not decrypt for its reader is treated as absent. Rotating the workspace key +(`set_encryption_key`) does not re-key the backups the way it re-encrypts the workspace's +secrets. The objects live under a prefix named by a generation +(`workspace_settings.ai_sessions_backup_generation`) that the rotation bumps in the +transaction committing the new key; once committed, the routes read and write under the new +generation's prefix, the answers name it (`backup_generation`, below; `storage_id` names the +storage and does not change), so every browser marks its sync rows stale and pushes its +sessions whole again there, and every older generation, which nothing writes to any more, is deleted off the +request at leisure (`windmill-api-workspaces/src/ai_session_backups.rs`). Sessions no browser +holds any more are lost. A generation is never reused, so no deletion, however late, can +touch live objects; a rotation that fails before its commit bumps nothing and deletes +nothing; two rotations racing serialize on the key row; the same key set again bumps +nothing. A rotation is rare, and the alternative, +rewriting every object in place while pushes, restarts, storage switches and further +rotations race the rewrite, is where the complexity would be; with this, nothing but the +current key ever reads an object. The +server builds every key from ids it validated +(`[A-Za-z0-9_-]{1,64}`) and the caller's own email; the client never names a key, and the +workspace storage permission rules are not consulted (the same stance as volumes). Only an +unscoped user token may reach the routes: a job token can carry an `on_behalf_of` identity and +every scoped token (guest, embed, app policy, MCP) was minted for something narrower. + +The backup is keyed by the email like the browser's own stores are (`userScopedDb` scopes +IndexedDB by it): a user whose email changes starts from an empty history on both sides, and +the objects under the old hash stay in the bucket unread. Carrying them over would need a +server-side re-key (decrypt with the old suffix, encrypt with the new, move every object) in the +email-change flow, which this design leaves out. + +An image is accepted only as a base64 data URL of at most 4 MB and stored verbatim, so it +serializes back into a pull answer at its stored size; anything JSON would escape could grow +several times and defeat the pull budget. + +`push` carries `owner`, the email the browser prepared the batch for, and the server refuses a +mismatch with 409: an in-place account switch must not file one user's sessions under another's +prefix. The client captures its user at flush start and checks every store handle's name +against it for the same reason. + +The feature is on wherever the workspace has primary storage, and off with +`ai_config.sessions_storage_disabled` (the `copilot_disabled` pattern: no migration, carried by +settings export and the CLI). A build without `parquet` has no routes (404), a workspace without +storage answers `enabled: false`; either turns the backup off for ten minutes, after which the +page asks again on its own (a flush for whatever is pending, and a restore), and the AI +settings page tells the mirror at once when the switch is saved there (the off state is +forgotten, the rows that went stale are marked again, a restore runs). + +## Conflicts and deletion + +Last write wins across devices. The head carries no manifest; `pull` lists the session's prefix +instead, so a stale device that renames or archives a session rewrites only the head and cannot +hide chats a newer device wrote. Two devices continuing the same chat still collide. + +Restore brings back only sessions the browser does not have (`importSessions` is write-if-absent, +and skips ids the user deleted in this page) and never overwrites or deletes a local one from +remote state. It covers the workspace and its forks together, and only once every one of them +that keeps backups has listed (a listing that failed leaves the family for the next page load +or workspace switch, or the copy that did list could be the stale one): a session listed by +two of them (moved between them, the old copy not yet removed, since that mark is the moving +browser's, which may never come back) is brought back from the copy that moved last (`epoch`, +the record's move count, which names the marker), the storage's own modification time +deciding between two of the same count, and not from the other, which would otherwise take +the id first and keep the later copy out for good. A workspace's records land together once +its pulls are done, and just before they do the whole family is listed again (members whose +backups were off included, since a move from another device can land in a workspace between +the first listings and the pulls; a family of one, with nowhere else for a copy to show up, is +not): a session a later copy of which showed up elsewhere is left, with the family, for the +next time. Only a user-initiated `deleteSession` removes the backup; the next push from +another device that still has the session is refused with `needs_whole` (nothing of it is +written), its row goes stale without a backoff, and that device's next flush sends the session +whole; the workspace-lifecycle +removals (`reconcileSessionsLifecycle`, `deleteSessionsForWorkspace`) leave it, so a session +dropped by a wrong reconcile comes back on the next restore. Objects of deleted workspaces stay +in the bucket. A session moved to another workspace is pushed whole into the new one, and once +that push has landed the copy in the old one gets a removal mark of its own, naming the storages holding +that copy (the row that knew is the new workspace's by then), retried independently until +each of them has answered, even when the old workspace's backups are off at the time (they +may hold the copy still). Filing the removal only after the new copy is acknowledged keeps the +session backed up somewhere at every point. + +A restore writes a session's artifacts and chats before its record, and records nothing for a +session whose pieces could not be written: recording it would let the next flush push the +half-empty local state over the backup. + +Every answer names the storage it came from (`storage_id`, a hash of what locates the objects, +endpoint, region and bucket, not the credentials, which rotate) and the backup generation a +key rotation bumps (`backup_generation`). A sync row records both, and a row naming another +storage or generation goes stale and its session is marked again: a workspace pointed at a +new bucket, or whose key was rotated, holds nothing, and the server looks nowhere else, so +the next flush carries the session whole. A switch leaves the old copy where it was, so the +row rewritten under the new storage records the old one (`alsoIn`, one entry per storage +the workspace was on), and a removal is done only once every storage holding a copy answered +it, whatever the generation (a rotation deleted the older generation's copy anyway): each +answer narrows the row to the storages still holding one, and the mark waits for them to +answer, so a switch back never brings a deleted session back. That includes the rows a flush has just written, when a later answer of the +same flush names another storage or the session was pushed in part on top of a row from the +old one; a session whose own parts were answered from different storages is not settled at +all. The listing a restore starts with runs the same check, so a storage switch is noticed at +the first push after it or on the next page load, whichever comes first. + +## Limits + +Push bodies are packed to about 8 MB (UTF-8 bytes as sent), at most 100 entries, 200 removals and +4000 pieces each (the server's caps, with 32 MB on the body, and 100 chats, 500 images or 1000 +deletes per entry, since every piece is an object-store call); an entry that +outgrows the target is split into chat-only parts (the artifacts and deletes on the last, the +head on the last too for an incremental push and on the first part, whatever it carries, for a +push of the session whole), and deletes +past the per-entry cap are carried over to the next push, which the session stays marked for. A chat above +16 MB or a session's artifacts above 8 MB are left out with a console warning; a chat that grew +past the cap after it was backed up has its copy deleted, so a restore never presents the old +transcript as the current one. A 413 fails only the sessions of that request. A +request the server refuses (any other 4xx but 404/403/409) stops the backup for the page but keeps +the marks and the sync state, so the next load tries again; a session the server reports it could +not store stays marked and is retried with backoff. A move files the old workspace's removal +mark before recording the new copy's row, so a mark that could not be written leaves the move +to be planned again. A workspace that answers `enabled: false` +marks its sync rows stale (the next push after storage returns carries every session whole, +since a new storage may be a new bucket) and leaves its dirty marks where they are (a move into +it must still remember the old copy); it keeps the removal marks of sessions that had been backed +up, so one deleted while backups are off does not come back once they are on, and drops the +removals of sessions never backed up from this browser, so a storage-less instance does not +collect one mark per deleted session forever. A user delete whose removal mark cannot be written +(localStorage full) is carried by the session's sync row instead (`removed`), which the flush +and the restore read like a mark; a session without a row yet (its first push may be in +flight) gets a row saying only that, and every row write keeps a removal filed meanwhile, so +the push's own row cannot erase it. Pull bodies are +capped at 64 KB. Pull answers up to 20 ids within a 32 MB +budget: a session's size is known from the listings before anything of it is read, one that +would not fit is deferred unless it is the first of the answer, in which case it comes in +pages: the answer carries what fits in key order (at least one object, so every page makes +progress) and names where the next picks up (`next`, a cursor the browser sends back as +`resume` with that session alone). Every page carries a fingerprint of the session's listing +(marker, keys, sizes, modification times, entity tags and versions, since a store reports +modification times coarsely and an object rewritten at the same size within that grain would +otherwise fingerprint the same) taken before anything of it is read, and the server +takes it again once the page is read: a page the backup moved under (a push landing object by +object) is read again, a few times, then answered as `moved`, and the browser starts the +session over on that or on two pages whose fingerprints differ. The browser writes each page's pieces as it arrives, over whatever +an earlier restore cut short had staged (the session is absent locally, so its pieces have no +local edits to keep, and the backup may have moved on), and the record, which is what makes +the session visible, only with the last page. A restore in progress keeps a staging row for +the session (the ids of every chat, image, artifact and version it wrote), which outlives it +if it is cut short; the next restore deletes the staged pieces the backup no longer has, by id +and never by clock, before the record lands (once the record is there no restore looks at the +session again, and a flush would push them back), and a prune that could not run leaves the +session, its pieces and its staging row for the restore after. A restore holds the user's tab +lock while it runs, so two tabs cannot each write the same absent session's pieces over the +other's, and runs only where Web Locks exist (a secure context: https, or localhost); on a plain +http origin the browser still backs up, and its sessions come back on a secure one. A +restore never writes an older record over a newer +one; between pages it holds nothing but the sync +row being assembled, whose chats also admit the images of a later page. Every page carries a +fingerprint of the session's listing (`listing`), taken before anything of the page is +listed or read, so an object landing after it is in the next page's; a session whose +fingerprint moved between two of its pages (a chat added by another device could sort before +the cursor and be missed) starts over, up to three times, then waits for the next restore. An object that grew +since the listing (a push replaced it) ends its page just before it and the answer names that +spot, so the next page sizes it anew rather than the session being imported without it. A pull sees every key of a session's listing but keeps the 5000 smallest +past its cursor (a page is defined by key order, and the store promises none), so a session +grown without bound by valid pushes cannot grow the answer's memory through its metadata +either; removing a prefix and a rotation's deletion stream their listings. `list` scans at most 50 000 index markers, keeps the newest 500 as it goes and answers with +them (`truncated` says when there were more); the restore takes 50 of them. Every read checks the object's size before buffering it: one larger than any push writes +(32 MB) is planted, whatever its listing said, and skipped, since whoever holds the bucket's +credentials can put anything at a predictable key; one larger than its listing said grew +since (a push replaced it) and ends its page, for the next pull to size anew. A dirty mark that cannot be written +(localStorage full) records its bump on the session's sync row instead (`extraV`, counted +with the mark's counter and kept by every row write, so a push in flight cannot retire it); +a session without a row yet (its first push in flight) keeps the bump in the page, and the +next row write takes it onto the row in the same transaction, so the row the push writes +cannot retire the mark with the bump unseen; every load's backfill marks again a session +without a row, with a stale one, or with one carrying bumps no push has covered. +A mark or removal for another user (a write that landed after a switch) reaches that user's +rows through a connection of its own, since the shared handle follows the current user. Nothing is read past the budget, whatever a session holds. A +restore takes the newest 50 sessions per workspace: every visible session gets a runtime, and +each runtime's history load reads the whole chat store. On CE the push checks the storage quota +and bumps usage by bytes written (an over-count on overwrites; the periodic recount settles it). diff --git a/frontend/src/lib/components/InstanceSettings.svelte b/frontend/src/lib/components/InstanceSettings.svelte index b1217b4328..f51e459f77 100644 --- a/frontend/src/lib/components/InstanceSettings.svelte +++ b/frontend/src/lib/components/InstanceSettings.svelte @@ -1084,11 +1084,12 @@ it with none, whether SSO logins evaluate an IdP groups claim (SAML or OIDC) and change a membership, the plan tier and quota shown when the execution meter is opened, whether app sandbox isolation is turned on, whether a step's workspace script is - edited from the flow editor, which skin approval steps are given, how data tables and - their migrations are set up and used, how often an empty workspace home is seen, how - often the home page’s create menu and hub-project picker are opened and from which - entry point, the name of any public hub project imported from the home page and how - far that import got, and whether a pre-approved trial offer was opened, last 30 days)
  • feature adoption (counts of which flow, script, trigger, worker and data table @@ -1149,11 +1150,12 @@ it with none, whether SSO logins evaluate an IdP groups claim (SAML or OIDC) and change a membership, the plan tier and quota shown when the execution meter is opened, whether app sandbox isolation is turned on, whether a step's workspace script is - edited from the flow editor, which skin approval steps are given, how data tables and - their migrations are set up and used, how often an empty workspace home is seen, how - often the home page’s create menu and hub-project picker are opened and from which - entry point, the name of any public hub project imported from the home page and how - far that import got, and whether a pre-approved trial offer was opened, last 30 days)
  • feature adoption (counts of which flow, script, trigger, worker and data table diff --git a/frontend/src/lib/components/copilot/chat/HistoryManager.svelte.ts b/frontend/src/lib/components/copilot/chat/HistoryManager.svelte.ts index 592b6217c8..4edac6dc0a 100644 --- a/frontend/src/lib/components/copilot/chat/HistoryManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/HistoryManager.svelte.ts @@ -1,9 +1,15 @@ -import { type DBSchema as IDBSchema, type IDBPDatabase } from 'idb' +import { + type DBSchema as IDBSchema, + type IDBPDatabase, + type IDBPTransaction, + type StoreNames +} from 'idb' import type { ChatJob, DisplayMessage } from './shared' import { expanded, messageDraft } from './chatDraft' import { createLongHash } from '$lib/editorLangUtils' import { userScopedDb, type UserScopedDbMigrateDeps } from '$lib/userScopedDb' -import { scopedKey } from '$lib/userScopedStorage' +import { emailOfScopedKey, scopedKey, scopedKeyFor } from '$lib/userScopedStorage' +import { markSessionDirty } from '$lib/components/sessions/sessionMirrorSignal' import type { ChatCompletionMessageParam } from 'openai/resources/index.mjs' import type { PersistedContextUsage } from './tokenUsage' import { IMAGE_OMITTED_PLACEHOLDER, type AttachedImage } from './imageUtils' @@ -14,8 +20,9 @@ import { randomUUID } from '$lib/utils/uuid' // shared browser. The bare name is also the legacy (pre-namespacing) DB, claimed // once on first login. const DB_NAME = 'copilot-chat-history' -// v3 adds the images blob store (replacing v2's short-lived toolImages store). -const DB_VERSION = 3 +// v3 adds the images blob store (replacing v2's short-lived toolImages store); v4 indexes +// chats by session. +const DB_VERSION = 4 /** Newest image blobs kept per chat; each is a bounded (≤1568px) data URL. */ const MAX_IMAGES_PER_CHAT = 30 /** Marks a persisted image whose bytes live in the `images` store. */ @@ -45,6 +52,7 @@ interface ChatSchema extends IDBSchema { // chats predating this feature. Persisted out-of-band like modifiedItems. backgroundJobs?: ChatJob[] } + indexes: { 'by-session': string } } // Image bytes, out-of-band from the chat record on purpose: the record is // re-cloned into IndexedDB on every saveChat, while a blob is written once @@ -64,9 +72,19 @@ interface ChatSchema extends IDBSchema { } } -function createChatStore(db: IDBPDatabase): void { - if (!db.objectStoreNames.contains('chats')) { - db.createObjectStore('chats', { keyPath: 'id' }) +/** A persisted chat, exactly as the store holds it (image refs, not bytes). */ +export type StoredChat = ChatSchema['chats']['value'] + +function createChatStore( + db: IDBPDatabase, + tx: IDBPTransaction[], 'versionchange'> +): void { + const chats = db.objectStoreNames.contains('chats') + ? tx.objectStore('chats') + : db.createObjectStore('chats', { keyPath: 'id' }) + // Lets the session backup find a session's chats without reading every record. + if (!chats.indexNames.contains('by-session')) { + chats.createIndex('by-session', 'sessionId') } // v2 briefly kept full-resolution tool screenshots in their own store; the // general blob store below covers them now. @@ -117,7 +135,9 @@ async function claimLegacyChatDb( { openDB, deleteDB }: UserScopedDbMigrateDeps ): Promise { if ((await scopedDb.count('chats')) > 0) return - const legacy = await openDB(DB_NAME, 1, { upgrade: createChatStore }) + const legacy = await openDB(DB_NAME, 1, { + upgrade: (db, _oldVersion, _newVersion, tx) => createChatStore(db, tx) + }) const legacyChats = await legacy.getAll('chats') if (legacyChats.length > 0) { const tx = scopedDb.transaction('chats', 'readwrite') @@ -156,6 +176,129 @@ export async function readChatModifiedItems(chatId: string): Promise(DB_NAME, { + version: DB_VERSION, + upgrade: createChatStore, + migrate: migrateLegacyChatDb +}) + +async function backupDb(email: string): Promise | undefined> { + const db = await backupDbh.whenReady() + return db && db.name === scopedKeyFor(DB_NAME, email) ? db : undefined +} + +/** Test-only: let go of the backup's handle so the next call opens the test's fresh + * IndexedDB rather than the connection a previous test left. */ +export function __resetBackupStoreForTesting(): void { + backupDbh.close() +} + +/** Ids of the chats tagged with this session, or undefined when the store is unavailable. */ +export async function listSessionChatIds( + sessionId: string, + email: string +): Promise { + const db = await backupDb(email) + if (!db) return undefined + return (await db.getAllKeysFromIndex('chats', 'by-session', sessionId)).map(String) +} + +export async function readStoredChat(id: string, email: string): Promise { + const db = await backupDb(email) + return db?.get('chats', id) +} + +/** Ids of the image blobs a chat owns, or undefined when the store is unavailable. */ +export async function listChatImageIds( + chatId: string, + email: string +): Promise { + const db = await backupDb(email) + if (!db) return undefined + return (await imageKeysForChat(db, chatId)).map(String) +} + +export async function readImageDataUrl(id: string, email: string): Promise { + const db = await backupDb(email) + return (await db?.get('images', id))?.dataUrl +} + +export interface RestoredImage { + id: string + chatId: string + dataUrl: string +} + +/** + * Write restored chats and image blobs, leaving any that already exist alone: a record + * this browser wrote since is newer than the backup it came from. False when the store + * could not be reached, which the caller must not record as a restore. + */ +export async function importStoredChats( + chats: StoredChat[], + images: RestoredImage[], + email: string, + overwrite = false +): Promise { + const db = await backupDb(email) + if (!db) return false + const tx = db.transaction(['chats', 'images'], 'readwrite') + const chatStore = tx.objectStore('chats') + const imageStore = tx.objectStore('images') + const savedAt = Date.now() + for (const image of images) { + if (overwrite || (await imageStore.getKey(image.id)) === undefined) { + await imageStore.put({ id: image.id, chatId: image.chatId, dataUrl: image.dataUrl, savedAt }) + } + } + // An overwrite never puts an older record over a newer one: without a cross-tab lock, + // another restore may have landed a newer backup's copy meanwhile. + for (const chat of chats) { + const existing = await chatStore.get(chat.id) + if (existing === undefined || (overwrite && existing.lastModified <= chat.lastModified)) { + await chatStore.put(chat) + } + } + await tx.done + return true +} + +/** Deletes these chats of the session (with their images) and these images: what an earlier + * restore staged for it and the backup no longer has. False when nothing could be deleted. */ +export async function pruneSessionChats( + sessionId: string, + chats: Set, + images: Set, + email: string +): Promise { + if (chats.size === 0 && images.size === 0) return true + const db = await backupDb(email) + if (!db) return false + try { + const tx = db.transaction(['chats', 'images'], 'readwrite') + const chatStore = tx.objectStore('chats') + const imageStore = tx.objectStore('images') + for (const chatId of await chatStore.index('by-session').getAllKeys(sessionId)) { + if (!chats.has(String(chatId))) continue + await chatStore.delete(chatId) + const keys = await imageStore + .index('by-chat') + .getAllKeys(IDBKeyRange.bound([chatId, -Infinity], [chatId, Infinity])) + for (const key of keys) await imageStore.delete(key) + } + for (const id of images) await imageStore.delete(id) + await tx.done + return true + } catch (err) { + console.error('Could not prune chats for session', err) + return false + } +} + export default class HistoryManager { // Per-instance handle to the shared per-user DB lifecycle. There is one // HistoryManager per AIChatManager (the singleton + one per session runtime), @@ -289,7 +432,10 @@ export default class HistoryManager { const snapshot = $state.snapshot(existing) const updated = { ...snapshot, sessionId } this.savedChats = { ...this.savedChats, [chatId]: updated } - await this.enqueueDbWrite((db) => db.put('chats', updated)) + await this.enqueueDbWrite(async (db) => { + await db.put('chats', updated) + markSessionDirty(sessionId, chatId, emailOfScopedKey(DB_NAME, db.name)) + }) } getPastChats() { @@ -565,6 +711,13 @@ export default class HistoryManager { const keep = this.keptImageIds(refs) await this.writeKeptImageBlobs(db, updatedChat.id, blobs, keep) await db.put('chats', updatedChat) + if (updatedChat.sessionId) { + markSessionDirty( + updatedChat.sessionId, + updatedChat.id, + emailOfScopedKey(DB_NAME, db.name) + ) + } // Best-effort: the record is already committed, so a failed cleanup // (e.g. a user switch closed this handle mid-op) must not turn a // successful save into a rejection — the orphans are reclaimed by @@ -589,6 +742,7 @@ export default class HistoryManager { } deletePastChat(id: string) { + const sessionId = this.savedChats[id]?.sessionId this.savedChats = Object.fromEntries( Object.entries(this.savedChats).filter(([key]) => key !== id) ) @@ -596,6 +750,7 @@ export default class HistoryManager { await db.delete('chats', id) const keys = await imageKeysForChat(db, id) await Promise.all(keys.map((key) => db.delete('images', key))) + if (sessionId) markSessionDirty(sessionId, id, emailOfScopedKey(DB_NAME, db.name)) }).catch((err) => console.error('Could not delete chat', err)) } diff --git a/frontend/src/lib/components/copilot/chat/artifacts/artifactsDB.ts b/frontend/src/lib/components/copilot/chat/artifacts/artifactsDB.ts index 94afac404e..e7ff4b1f60 100644 --- a/frontend/src/lib/components/copilot/chat/artifacts/artifactsDB.ts +++ b/frontend/src/lib/components/copilot/chat/artifacts/artifactsDB.ts @@ -2,6 +2,8 @@ // active chat's rotation, so chatId-keying would drop artifacts on each new conversation. import { type DBSchema as IDBSchema, type IDBPObjectStore, type IDBPTransaction } from 'idb' import { userScopedDb } from '$lib/userScopedDb' +import { emailOfScopedKey, scopedKeyFor } from '$lib/userScopedStorage' +import { markSessionDirty } from '$lib/components/sessions/sessionMirrorSignal' export type ArtifactKind = 'md' | 'html' @@ -89,9 +91,11 @@ interface ArtifactsSchema extends IDBSchema { } } +const ARTIFACTS_DB = 'copilot-artifacts' + // User-scoped like the chat-history store these are keyed against: no cross-user // co-residency on a shared browser. -const dbh = userScopedDb('copilot-artifacts', { +const dbh = userScopedDb(ARTIFACTS_DB, { version: 2, // Runs for a fresh database and for the v1 upgrade alike, so create each store only // when it is missing. @@ -118,11 +122,69 @@ export async function putArtifact(artifact: PersistedArtifact): Promise { // A rejected write (most likely QuotaExceededError) leaves the artifact usable for the // session but unpersisted — degrade like the reads rather than throwing at the caller. await db.put('items', artifact) + markSessionDirty(artifact.sessionId, undefined, emailOfScopedKey(ARTIFACTS_DB, db.name)) } catch (err) { console.error('Could not persist artifact', err) } } +/** A session's artifacts with their history, or undefined when the store is unavailable + * or no longer the named user's (see `readStoredSessions`). */ +export async function readSessionArtifacts( + sessionId: string, + email: string +): Promise<{ items: PersistedArtifact[]; versions: ArtifactVersion[] } | undefined> { + const db = await getDB() + if (!db || db.name !== scopedKeyFor(ARTIFACTS_DB, email)) return undefined + try { + const items = await db.getAllFromIndex('items', 'by-session', sessionId) + const versions = ( + await Promise.all(items.map((i) => db.getAllFromIndex('versions', 'by-artifact', i.id))) + ).flat() + return { items, versions } + } catch (err) { + console.error('Could not read artifacts', err) + return undefined + } +} + +/** Write restored artifacts and snapshots, leaving any that already exist alone. Reports + * whether they are all in the store now: unlike the other writes here, a caller records the + * restore as done on the strength of this answer. */ +export async function importArtifacts( + items: PersistedArtifact[], + versions: ArtifactVersion[], + email: string, + overwrite = false +): Promise { + if (items.length === 0 && versions.length === 0) return true + const db = await getDB() + if (!db || db.name !== scopedKeyFor(ARTIFACTS_DB, email)) return false + try { + const tx = db.transaction(['items', 'versions'], 'readwrite') + const itemStore = tx.objectStore('items') + const versionStore = tx.objectStore('versions') + // An overwrite never puts an older record over a newer one: without a cross-tab lock, + // another restore may have landed a newer backup's copy meanwhile. + for (const item of items) { + const existing = await itemStore.get(item.id) + if (existing === undefined || (overwrite && existing.updatedAt <= item.updatedAt)) { + await itemStore.put(item) + } + } + for (const version of versions) { + if (overwrite || (await versionStore.getKey(version.key)) === undefined) { + await versionStore.put(version) + } + } + await tx.done + return true + } catch (err) { + console.error('Could not import artifacts', err) + return false + } +} + export async function getArtifact(id: string): Promise { const db = await getDB() if (!db) return undefined @@ -273,7 +335,11 @@ export async function mutateArtifact( reportFailure = false abort() } - return { outcome: await settled, artifact: edit.artifact } + const outcome = await settled + if (outcome === 'saved' && db) { + markSessionDirty(edit.artifact.sessionId, undefined, emailOfScopedKey(ARTIFACTS_DB, db.name)) + } + return { outcome, artifact: edit.artifact } } /** @@ -342,14 +408,47 @@ export async function deleteArtifact(id: string): Promise { if (!db) return try { const tx = db.transaction(['items', 'versions'], 'readwrite') - await tx.objectStore('items').delete(id) + const items = tx.objectStore('items') + const sessionId = (await items.get(id))?.sessionId + await items.delete(id) await deleteVersionsIn(tx.objectStore('versions'), id) await tx.done + if (sessionId) markSessionDirty(sessionId, undefined, emailOfScopedKey(ARTIFACTS_DB, db.name)) } catch (err) { console.error('Could not delete artifact', err) } } +/** Deletes these artifacts of the session (with their versions) and these versions: what + * an earlier restore staged for it and the backup no longer has. False when nothing could + * be deleted. */ +export async function pruneSessionArtifacts( + sessionId: string, + itemIds: Set, + versionKeys: Set, + email: string +): Promise { + if (itemIds.size === 0 && versionKeys.size === 0) return true + const db = await getDB() + if (!db || db.name !== scopedKeyFor(ARTIFACTS_DB, email)) return false + try { + const tx = db.transaction(['items', 'versions'], 'readwrite') + const items = tx.objectStore('items') + const versions = tx.objectStore('versions') + for (const id of await items.index('by-session').getAllKeys(sessionId)) { + if (!itemIds.has(String(id))) continue + await items.delete(id) + await deleteVersionsIn(versions, String(id)) + } + for (const key of versionKeys) await versions.delete(key) + await tx.done + return true + } catch (err) { + console.error('Could not prune artifacts for session', err) + return false + } +} + export async function deleteArtifactsForSession(sessionId: string): Promise { const db = await getDB() if (!db) return diff --git a/frontend/src/lib/components/sessions/sessionMirror.svelte.ts b/frontend/src/lib/components/sessions/sessionMirror.svelte.ts new file mode 100644 index 0000000000..b1ac4bcd06 --- /dev/null +++ b/frontend/src/lib/components/sessions/sessionMirror.svelte.ts @@ -0,0 +1,1644 @@ +// Lazily backs the browser's AI sessions up to their workspace's object storage, and +// restores the ones this browser does not have. +// +// IndexedDB stays the store every write lands in; the funnels there only mark a session +// dirty (sessionMirrorSignal). A flush runs once the marks have been quiet for a while, +// bounded by a maximum delay so a long turn still gets backed up part-way, and sends +// batched requests per workspace carrying only the pieces whose marker moved +// (sessionMirrorPlan). Marks are persisted, so a crash leaves them for the next load. +// +// What a flush is for is decided per workspace: `enabled: false` (no storage, or the +// admin switch) turns it off for ten minutes, after which the page asks again on its own, +// and at once when the switch is saved from this page (`backupSettingsChanged`). +import { BROWSER } from 'esm-env' +import { get } from 'svelte/store' +import { openDB, type DBSchema, type IDBPDatabase } from 'idb' +import { + AiService, + ApiError, + type AISessionBackup, + type AISessionBackupCursor, + type AISessionBackupImage, + type AISessionBackupPush +} from '$lib/gen' +import { userWorkspaces } from '$lib/stores' +import { userScopedDb } from '$lib/userScopedDb' +import { getCurrentUserEmail, onUserChange, scopedKey, scopedKeyFor } from '$lib/userScopedStorage' +import { logFeatureUsage } from '$lib/utils/featureUsage' +import { randomUUID } from '$lib/utils/uuid' +import { workspaceRootId } from './sessionScope.svelte' +import { onMirrorSignal } from './sessionMirrorSignal' +import { + importSessions, + isSessionTombstoned, + readStoredSessions, + sessionState, + type Session +} from './sessionState.svelte' +import { + importStoredChats, + pruneSessionChats, + listChatImageIds, + listSessionChatIds, + readImageDataUrl, + readStoredChat, + type RestoredImage, + type StoredChat +} from '../copilot/chat/HistoryManager.svelte' +import { + importArtifacts, + pruneSessionArtifacts, + readSessionArtifacts, + type ArtifactVersion, + type PersistedArtifact +} from '../copilot/chat/artifacts/artifactsDB' +import { + artifactsFingerprint, + headSig, + jsonBytes, + operationsOf, + planSessionPush, + splitEntry, + type ChatSnapshot, + type MirrorSyncState, + type PlannedPush, + type PushBody +} from './sessionMirrorPlan' + +/** A flush waits for the marks to go quiet this long. */ +const QUIET_MS = 15_000 +/** ...but never longer than this after the first unflushed mark. */ +const MAX_DELAY_MS = 120_000 +const STARTUP_DELAY_MS = 10_000 +const RETRY_MIN_MS = 30_000 +const RETRY_MAX_MS = 600_000 +/** Requests are packed up to about this many bytes; the server accepts four times that. */ +const REQUEST_TARGET_BYTES = 8 * 1024 * 1024 +/** The server's caps per request and per entry. */ +const MAX_ENTRIES_PER_REQUEST = 100 +const MAX_REMOVED_PER_REQUEST = 200 +const MAX_IMAGES_PER_ENTRY = 500 +const MAX_OPERATIONS_PER_REQUEST = 4000 +/** A chat or a session's artifacts beyond this are left out of the backup rather than + * sent: with the record and the deletes riding along, the largest entry stays well under + * the server's 32 MB body cap. */ +const MAX_CHAT_BYTES = 16 * 1024 * 1024 +const MAX_ARTIFACTS_BYTES = 8 * 1024 * 1024 +const PULL_BATCH = 5 +/** Newest sessions restored per workspace: every visible session gets a runtime, and each + * runtime's history load reads the whole chat store. */ +const RESTORE_MAX = 50 +const PENDING_PREFIX = 'windmill_sessions_mirror_pending' +const SYNC_DB = 'windmill-sessions-mirror' + +interface MirrorSchema extends DBSchema { + sync: { key: string; value: MirrorSyncState } +} + +function createSyncStore(db: IDBPDatabase): void { + if (!db.objectStoreNames.contains('sync')) db.createObjectStore('sync', { keyPath: 'id' }) +} + +const syncDbh = userScopedDb(SYNC_DB, { version: 1, upgrade: createSyncStore }) + +async function syncDb(email: string) { + const db = await syncDbh.whenReady() + return db && db.name === scopedKeyFor(SYNC_DB, email) ? db : undefined +} + +// --- Pending marks --- +// +// One localStorage key per mark, shared by every tab of the user: a dirty mark holds a +// counter bumped on every write, a removal mark the workspace to remove from and, for a +// session that moved to another workspace, the storages holding the old copy (the row that +// knew is the new workspace's by then). Keying each mark on its own is what lets two tabs +// mark different sessions at the same time without one rewriting the other's mark away, as +// a single JSON blob would. + +interface PendingMarks { + dirty: { id: string; v: number }[] + removed: { id: string; ws?: string; key: string; storages?: string[] }[] +} + +function pendingBase(): string | undefined { + return scopedKey(PENDING_PREFIX) +} + +/** The marks of the user a write landed for: the current user unless the signal says + * otherwise (its store's user, when the user changed while the write was pending). */ +function pendingBaseFor(email: string | undefined): string | undefined { + return email ? scopedKeyFor(PENDING_PREFIX, email) : pendingBase() +} + +function dirtyKey(base: string, sessionId: string): string { + return `${base}::d::${sessionId}` +} + +function removedKey(base: string, sessionId: string, ws: string | undefined): string { + return `${base}::r::${sessionId}::${ws ?? ''}` +} + +function readPending(): PendingMarks { + const marks: PendingMarks = { dirty: [], removed: [] } + const base = pendingBase() + if (!base) return marks + try { + for (let i = 0; i < localStorage.length; i++) { + const key = localStorage.key(i) + if (!key || !key.startsWith(`${base}::`)) continue + const rest = key.slice(base.length + 2) + if (rest.startsWith('d::')) { + const v = Number(localStorage.getItem(key)) + marks.dirty.push({ id: rest.slice(3), v: Number.isFinite(v) ? v : 0 }) + } else if (rest.startsWith('r::')) { + const [id, ws] = rest.slice(3).split('::') + marks.removed.push({ id, ws: ws || undefined, key, storages: storagesOf(key) }) + } + } + } catch (e) { + console.error('Could not read session backup marks', e) + } + // Marks this page could not write: their rows carry the bumps, or this page does until + // a row exists (see `bumpViaSyncRow`); the counter counts with those. + for (const [id, bumps] of unwritableMarks) { + const mark = marks.dirty.find((d) => d.id === id) + if (mark) mark.v += bumps + else marks.dirty.push({ id, v: bumps }) + } + return marks +} + +/** False when the mark could not be written (storage full): the caller must carry the + * change some other way. */ +function bumpDirty(sessionId: string, email?: string): boolean { + const base = pendingBaseFor(email) + if (!base) return false + try { + const key = dirtyKey(base, sessionId) + const v = Number(localStorage.getItem(key) ?? '0') + localStorage.setItem(key, String((Number.isFinite(v) ? v : 0) + 1)) + return true + } catch (e) { + console.error('Could not persist session backup mark', e) + return false + } +} + +/** The durable fallback for a dirty mark that could not be written: the bump goes on the + * sync row, where a flush in flight cannot lose it (`writeSync` keeps it). A session with + * no row yet (its first push in flight, say) keeps it in this page until a row is written, + * which takes it over (see `writeSync`): the row the push writes would otherwise retire the + * mark with the bump unseen. Another user's session with no row is left to that user's next + * load, whose backfill marks every session without a row. */ +async function bumpViaSyncRow(id: string, email = getCurrentUserEmail()): Promise { + if (!email) return + let carried = false + await updateSyncRow(id, email, (row) => { + if (!row) return undefined + carried = true + return { ...row, extraV: (row.extraV ?? 0) + 1 } + }) + if (!carried && email === getCurrentUserEmail()) { + unwritableMarks.set(id, (unwritableMarks.get(id) ?? 0) + 1) + } +} + +/** Reads a row and writes what `update` makes of it, in the store of `email`: through the + * shared handle when that is the current user, and through a connection of its own + * otherwise, since the shared handle follows the current user and a write that landed + * after a switch must still reach the store it belongs to. */ +async function updateSyncRow( + id: string, + email: string, + update: (row: MirrorSyncState | undefined) => MirrorSyncState | undefined +): Promise { + if (email === getCurrentUserEmail()) { + const next = update(await readSync(id, email)) + if (next) await writeSync([next], email) + return + } + let db: IDBPDatabase | undefined + try { + db = await openDB(scopedKeyFor(SYNC_DB, email), 1, { upgrade: createSyncStore }) + const next = update(await db.get('sync', id)) + if (next) await db.put('sync', next) + } catch (e) { + console.error('Could not update the session backup state of another user', e) + } finally { + db?.close() + } +} + +/** Drop the dirty mark of a session gone from the store, the one case nothing can bump + * again. Every other mark stays: one whose push landed is retired through `flushedV` on + * the sync row (two localStorage calls cannot compare-and-delete, and a bump landing + * between them would be lost), and a draft's or an off workspace's waits its turn. */ +function dropDirty(sessionId: string): void { + const base = pendingBase() + if (base) removeKey(dirtyKey(base, sessionId)) +} + +/** The storages a removal mark names, when it does. */ +function storagesOf(key: string): string[] | undefined { + try { + const parsed: unknown = JSON.parse(localStorage.getItem(key) ?? '') + if (Array.isArray(parsed) && parsed.every((s) => typeof s === 'string')) return parsed + } catch {} + return undefined +} + +/** False when the mark could not be written (storage full): the caller must not act as if + * the removal were scheduled. */ +function addRemoved( + sessionId: string, + ws: string | undefined, + dropDirty: boolean, + email?: string, + storages?: string[] +): boolean { + const base = pendingBaseFor(email) + if (!base) return false + try { + if (dropDirty) localStorage.removeItem(dirtyKey(base, sessionId)) + localStorage.setItem( + removedKey(base, sessionId, ws), + storages && storages.length > 0 ? JSON.stringify(storages) : '1' + ) + return true + } catch (e) { + console.error('Could not persist session backup mark', e) + return false + } +} + +function removeKey(key: string): void { + try { + localStorage.removeItem(key) + } catch {} +} + +// --- Scheduling --- + +let quietTimer: ReturnType | undefined +let maxTimer: ReturnType | undefined +let retryTimer: ReturnType | undefined +let retryAt = 0 +let retryMs = RETRY_MIN_MS +/** + * Workspaces whose storage answered this page load. `off`: nowhere to keep backups. + * `refused`: the server rejected what this page sends; nothing more is sent for the page, + * but the marks and the sync state stay, so the next load tries again. + */ +const wsState = new Map() +/** Backups an admin turns on again elsewhere are noticed `OFF_RETRY_MS` after the workspace + * was found off: a timer per off workspace forgets the state then and asks again, so a + * pending mark left for the workspace does not wait for something else to flush. The + * admin's own page hears of the switch at once (see `backupSettingsChanged`). */ +const offTimers = new Map>() +const OFF_RETRY_MS = 10 * 60_000 +let offRetryMs = OFF_RETRY_MS + +function markOff(ws: string): void { + wsState.set(ws, 'off') + clearTimeout(offTimers.get(ws)) + offTimers.set( + ws, + setTimeout(() => { + offTimers.delete(ws) + if (wsState.get(ws) === 'off') wsState.delete(ws) + restoredWorkspaces.delete(ws) + void enqueue(flush) + restoreSessionBackups(ws) + }, offRetryMs) + ) +} + +function forgetOff(ws: string): void { + clearTimeout(offTimers.get(ws)) + offTimers.delete(ws) + if (wsState.get(ws) === 'off') wsState.delete(ws) +} + +function clearOffTimers(): void { + for (const timer of offTimers.values()) clearTimeout(timer) + offTimers.clear() +} + +function isOff(ws: string): boolean { + return wsState.get(ws) === 'off' +} +let backfilled = false +/** Sessions of the current user whose dirty mark could not be written this page. */ +/** Sessions whose dirty mark localStorage refused this page, with the bumps of theirs no + * sync row could take yet. */ +let unwritableMarks = new Map() +const restoredWorkspaces = new Set() + +// One flush or restore at a time in this tab; each reads the marks fresh. +let chain: Promise = Promise.resolve() +function enqueue(fn: () => Promise): Promise { + const run = chain.then(fn, fn) + chain = run.catch((e) => console.error('Session backup failed', e)) + return run +} + +function clearTimers(): void { + clearTimeout(quietTimer) + clearTimeout(maxTimer) + quietTimer = undefined + maxTimer = undefined +} + +function scheduleFlush(): void { + clearTimeout(quietTimer) + quietTimer = setTimeout(runFlush, QUIET_MS) + maxTimer ??= setTimeout(runFlush, MAX_DELAY_MS) +} + +function runFlush(): void { + clearTimers() + void enqueue(flush) +} + +function backOff(): void { + retryAt = Date.now() + retryMs + retryMs = Math.min(retryMs * 2, RETRY_MAX_MS) + clearTimeout(retryTimer) + retryTimer = setTimeout(runFlush, retryAt - Date.now()) +} + +/** Serialize with the other tabs of the same user where the browser lets us; on plain + * http there is no lock, and two tabs at worst upload the same bytes twice. */ +/** One tab of the user at a time in the flush and the restore. A flush finding the lock + * taken reschedules itself; a restore waits its turn, since two tabs restoring the same + * absent session would each write its pieces over the other's. */ +function webLocks(): LockManager | undefined { + return typeof navigator === 'undefined' ? undefined : (navigator as { locks?: LockManager }).locks +} + +function hasWebLocks(): boolean { + return webLocks() !== undefined +} + +async function withUserLock(email: string, fn: () => Promise, wait = false): Promise { + const locks = webLocks() + if (!locks) return fn() + await locks.request(`wm-ai-sessions-mirror::${email}`, { ifAvailable: !wait }, async (lock) => { + if (lock) await fn() + // The other tab's flush read the marks before this one's were written: try again + // once it is done, rather than wait for the next write or load. + else scheduleFlush() + }) +} + +// --- Flush --- + +function statusOf(e: unknown): number | undefined { + return e instanceof ApiError ? e.status : undefined +} + +async function readSync(id: string, email: string): Promise { + return (await syncDb(email))?.get('sync', id) +} + +async function allSyncRows(email: string): Promise { + return (await (await syncDb(email))?.getAll('sync')) ?? [] +} + +/** The durable fallback for a user delete whose localStorage mark could not be written. + * A session with a row was backed up; one without may have its first push in flight, so + * it gets a row saying only that, which the push's own row write keeps (`writeSync`). An + * unsent draft gets nothing: it was never pushed. */ +async function removeViaSyncRow( + id: string, + ws: string | undefined, + email = getCurrentUserEmail() +): Promise { + if (!email) return + await updateSyncRow(id, email, (row) => + row + ? { ...row, removed: true } + : ws + ? { id, ws, head: '', chats: {}, images: {}, removed: true } + : undefined + ) +} + +/** Writes rows whole, except that a removal filed on a row meanwhile survives: the flush + * writes a session's row from state it read before the push, and the user may have deleted + * the session in between. */ +/** A row that says where else the copy is (`alsoIn`) is believed; one that does not + * inherits it, plus the storage the row it replaces was on when that is another one of this + * workspace's: the copy there stays where it was. */ +async function writeSync(states: MirrorSyncState[], email: string): Promise { + const db = await syncDb(email) + if (!db || states.length === 0) return + // Bumps this page kept for want of a row (see `bumpViaSyncRow`) move onto the row now; + // what arrives while the write is in flight stays counted here. + const taken = new Map() + const tx = db.transaction('sync', 'readwrite') + for (const s of states) { + const cur = await tx.store.get(s.id) + const removed = s.removed || cur?.removed + const kept = email === getCurrentUserEmail() ? (unwritableMarks.get(s.id) ?? 0) : 0 + if (kept > 0) taken.set(s.id, kept) + const extraV = Math.max(s.extraV ?? 0, cur?.extraV ?? 0) + kept + const alsoIn = new Set(s.alsoIn ?? []) + if (s.alsoIn === undefined && cur?.ws === s.ws) { + for (const id of cur.alsoIn ?? []) alsoIn.add(id) + if (cur.storageId !== undefined) alsoIn.add(cur.storageId) + } + if (s.storageId !== undefined) alsoIn.delete(s.storageId) + const { alsoIn: _, ...rest } = s + await tx.store.put({ + ...rest, + ...(removed ? { removed: true } : {}), + ...(extraV > 0 ? { extraV } : {}), + ...(alsoIn.size > 0 ? { alsoIn: [...alsoIn] } : {}) + }) + } + await tx.done + for (const [id, n] of taken) unwritableMarks.set(id, (unwritableMarks.get(id) ?? 0) - n) +} + +async function deleteSync(ids: string[], email: string): Promise { + const db = await syncDb(email) + if (!db || ids.length === 0) return + const tx = db.transaction('sync', 'readwrite') + for (const id of ids) await tx.store.delete(id) + await tx.done +} + +/** A workspace's storage went away: what was pushed there can no longer be trusted to be + * where a later storage looks, so every session goes whole on the next push. The rows stay + * (stale) so a removal still knows a backup existed. */ +async function staleWorkspaceSync(ws: string, email: string): Promise { + const db = await syncDb(email) + if (!db) return + const rows = (await db.getAll('sync')).filter((s) => s.ws === ws && !s.stale) + await writeSync( + rows.map((s) => ({ ...s, stale: true })), + email + ) +} + +/** The workspace's live rows recorded against another storage than the one the server + * answers from: they describe objects it no longer looks at (a new bucket starts empty). */ +function foreignRows( + ws: string, + storageId: string, + generation: number, + rows: Iterable +): MirrorSyncState[] { + return [...rows].filter( + (row) => + row.ws === ws && + !row.stale && + !row.removed && + !row.staging && + (row.storageId !== storageId || (row.generation ?? 0) !== generation) + ) +} + +/** Stale rows plan like no row at all, and the mark makes the next flush pick them up. */ +async function markStale(rows: MirrorSyncState[], email: string): Promise { + await writeSync( + rows.map((row) => ({ ...row, stale: true })), + email + ) + for (const row of rows) bumpDirty(row.id) +} + +/** Sessions this browser has backed up nothing of yet, whose backup went stale, or whose + * row carries bumps (localStorage refused their marks) no push has covered: everything + * committed to a workspace gets a mark, once per page load. */ +async function backfillMarks(marks: PendingMarks, email: string): Promise { + if (backfilled) return true + const sessions = await readStoredSessions(email) + const db = await syncDb(email) + if (!sessions || !db) return false + backfilled = true + const rows = new Map((await db.getAll('sync')).map((s) => [s.id, s])) + const marked = new Set(marks.dirty.map((d) => d.id)) + for (const s of sessions) { + const row = rows.get(s.id) + // A restored row has neither counter: only bumps it carries make it owed. + const owed = + !row || row.stale || ((row.extraV ?? 0) > 0 && (row.flushedV ?? -1) < (row.extraV ?? 0)) + if (s.workspace_id && owed && !marked.has(s.id)) { + // The counter as the next `readPending` will see it: the mark's, or this page's + // when none could be written, so what this flush retires is what a later bump + // counts from. + let v = 1 + if (!bumpDirty(s.id)) { + if (!unwritableMarks.has(s.id)) unwritableMarks.set(s.id, 0) + v = unwritableMarks.get(s.id) ?? 0 + } + marks.dirty.push({ id: s.id, v }) + } + } + return true +} + +/** Read what the stores hold for a dirty session and plan its push. `undefined` when the + * session has nowhere to go (unsent), `unavailable` when a store could not be read. */ +async function planFor( + session: Session, + sync: MirrorSyncState | undefined, + email: string +): Promise { + const chatIds = await listSessionChatIds(session.id, email) + if (!chatIds) return 'unavailable' + const chats: ChatSnapshot[] = [] + for (const id of chatIds) { + const record = await readStoredChat(id, email) + if (!record) continue + const imageIds = (await listChatImageIds(id, email)) ?? [] + if (jsonBytes(record) > MAX_CHAT_BYTES) { + console.warn(`AI session chat ${id} is too large to back up; leaving it out`) + chats.push({ id, lastModified: record.lastModified, imageIds, omitted: true }) + continue + } + chats.push({ id, lastModified: record.lastModified, record, imageIds }) + } + let artifacts = await readSessionArtifacts(session.id, email) + if (!artifacts) return 'unavailable' + if (jsonBytes(artifacts) > MAX_ARTIFACTS_BYTES) { + console.warn(`AI session ${session.id} artifacts are too large to back up; leaving them out`) + artifacts = { items: [], versions: [] } + } + return planSessionPush({ session, chats, artifacts, sync }) +} + +interface WorkspaceWork { + items: { session: Session; v: number; sync?: MirrorSyncState }[] + /** `key` is the localStorage mark; absent when the removal rides on the sync row. + * `storageId` and `alsoIn` are the storages holding a copy, when a sync row says: a + * removal is done only once each of them answered it. */ + removed: { id: string; key?: string; storageId?: string; alsoIn?: string[] }[] +} + +type SendStatus = 'ok' | 'off' | 'refused' | 'abort' | 'transient' + +interface WorkspaceOutcome { + status: SendStatus + /** The storage and backup generation the server answered from, once it answered. */ + storageId?: string + generation?: number + /** Sessions every part of which the server stored. */ + settled: { + id: string + v: number + next: MirrorSyncState + removeFrom?: string + carried: boolean + storageId?: string + generation?: number + }[] + /** Marks with nothing behind them (an unsent draft, a session gone from the store). */ + dropped: { id: string; v: number }[] + /** Sessions the server holds no head of any more (another device removed the backup): + * their rows go stale, so the next flush sends them whole. */ + needsWhole: string[] + /** Removal marks the server carried out. */ + removedDone: { id: string; key?: string }[] + /** Removals one storage carried out while others still hold a copy: the row narrows to + * those, and the mark waits for them to answer. */ + removedFrom: { id: string; key?: string; remaining: string[] }[] + /** Some session the server could not store. */ + anyFailed: boolean + /** Some store could not be read; its marks stay for a later flush. */ + unavailable: boolean +} + +/** + * Plan and send a workspace's sessions one at a time, filling requests of about + * REQUEST_TARGET_BYTES as it goes, so a first backfill never holds more than one + * request's worth of records and images at once. Removals go first, in their own + * request(s). + */ +async function pushWorkspace( + ws: string, + work: WorkspaceWork, + email: string +): Promise { + const out: WorkspaceOutcome = { + status: 'ok', + settled: [], + dropped: [], + needsWhole: [], + removedDone: [], + removedFrom: [], + anyFailed: false, + unavailable: false + } + // Parts of a session still to be acknowledged, and the sessions the server refused. + // `complete` once every part of the session has been appended: a session whose first + // part is still to come when a request fails has nothing behind it yet. + const attempted = new Map< + string, + { + v: number + next: MirrorSyncState + parts: number + complete: boolean + removeFrom?: string + carried: boolean + storageId?: string + generation?: number + /** The storage and generation the last answer for this session came from. */ + answered?: string + } + >() + const failed = new Set() + const headless = new Set() + let current: PushBody | undefined + let size = 0 + let ops = 0 + + const send = async (body: PushBody): Promise => { + if (getCurrentUserEmail() !== email) return 'abort' + let res + try { + res = await AiService.pushAiSessionBackups({ workspace: ws, requestBody: body }) + } catch (e) { + const status = statusOf(e) + // 404: a build without object storage. 403: nothing this token may back up. + if (status === 404 || status === 403) return 'off' + if (status === 409) return 'abort' + // Too large is a fact about the sessions in this body, not the workspace: they + // stay marked (and retried with backoff), the others go on. + if (status === 413) { + console.error('Session backup push too large', e) + for (const entry of body.sessions) failed.add(entry.id) + for (const id of body.removed ?? []) failed.add(id) + return 'ok' + } + if (status !== undefined && status < 500 && status !== 429) { + console.error('Session backup push refused', e) + return 'refused' + } + console.warn('Session backup push failed, retrying later', e) + return 'transient' + } + if (!res.enabled) return 'off' + out.storageId = res.storage_id + out.generation = res.backup_generation + const answered = `${res.storage_id}:${res.backup_generation}` + const errors = new Set() + for (const r of res.results) { + if (r.error) { + console.warn(`Session backup of ${r.id} failed: ${r.error}`) + errors.add(r.id) + } else if (r.needs_whole) { + // The part rode on a head the storage no longer has: not a failure to back + // off from, but nothing to settle either. + headless.add(r.id) + out.needsWhole.push(r.id) + } + } + for (const entry of body.sessions) { + const a = attempted.get(entry.id) + if (!a) continue + a.parts -= 1 + // Parts answered from different storages sit in different buckets: nothing to + // settle, the session goes again whole. + if (a.answered !== undefined && a.answered !== answered) failed.add(entry.id) + a.answered = answered + a.storageId = res.storage_id + a.generation = res.backup_generation + if (errors.has(entry.id)) failed.add(entry.id) + } + for (const id of body.removed ?? []) { + if (errors.has(id)) failed.add(id) + else { + const mark = work.removed.find((r) => r.id === id) + if (!mark) continue + const holding = new Set( + [mark.storageId, ...(mark.alsoIn ?? [])].filter((s): s is string => s !== undefined) + ) + // Answered from a storage holding no copy: the copies are still where they + // were, and the mark waits for those storages to answer. + if (holding.size === 0 || res.storage_id === undefined) out.removedDone.push(mark) + else if (holding.has(res.storage_id)) { + holding.delete(res.storage_id) + if (holding.size === 0) out.removedDone.push(mark) + else out.removedFrom.push({ id, key: mark.key, remaining: [...holding] }) + } + } + } + return 'ok' + } + const flushCurrent = async (): Promise => { + if (!current) return 'ok' + const body = current + current = undefined + size = 0 + ops = 0 + return send(body) + } + const append = async (entry: AISessionBackupPush): Promise => { + const bytes = jsonBytes(entry) + const entryOps = operationsOf(entry) + if ( + current && + (current.sessions.length >= MAX_ENTRIES_PER_REQUEST || + ops + entryOps > MAX_OPERATIONS_PER_REQUEST || + size + bytes > REQUEST_TARGET_BYTES) + ) { + const status = await flushCurrent() + if (status !== 'ok') return status + } + // A part the server refused holds the rest of its session back: the head rides on + // the last part, and would list a session missing a piece. + if (failed.has(entry.id)) return 'ok' + current ??= { owner: email, sessions: [] } + current.sessions.push(entry) + size += bytes + ops += entryOps + const a = attempted.get(entry.id) + if (a) a.parts += 1 + return 'ok' + } + const finish = (status: SendStatus): WorkspaceOutcome => { + out.status = status + for (const [id, a] of attempted) { + if (a.complete && a.parts === 0 && !failed.has(id) && !headless.has(id)) { + out.settled.push({ + id, + v: a.v, + next: a.next, + removeFrom: a.removeFrom, + carried: a.carried, + storageId: a.storageId, + generation: a.generation + }) + } + } + out.anyFailed = failed.size > 0 + return out + } + + for (let i = 0; i < work.removed.length; i += MAX_REMOVED_PER_REQUEST) { + const chunk = work.removed.slice(i, i + MAX_REMOVED_PER_REQUEST) + const status = await send({ owner: email, sessions: [], removed: chunk.map((r) => r.id) }) + if (status !== 'ok') return finish(status) + } + + for (const item of work.items) { + if (getCurrentUserEmail() !== email) return finish('abort') + const plan = await planFor(item.session, item.sync, email) + if (plan === 'unavailable') { + out.unavailable = true + continue + } + if (!plan) { + out.dropped.push({ id: item.session.id, v: item.v }) + continue + } + const nothingToSend = !plan.entry && plan.images.length === 0 + // A move: the copy in the old workspace is filed for removal once this push has + // landed (see `settled`), never before, so the session is backed up somewhere at + // every point. + // A session with nothing to send gets no answer to name its storage: it keeps the + // one its row has, which the storage check below then judges like any other. + attempted.set(item.session.id, { + v: item.v, + next: plan.next, + parts: 0, + complete: nothingToSend, + removeFrom: plan.removeFrom, + carried: plan.carried, + storageId: item.sync?.storageId, + generation: item.sync?.generation + }) + if (nothingToSend) continue + // A push of the session whole names itself on every part and opens with its head on + // the first, whichever that is: the server replaces the backup on that part, lists + // the session by the last, and meanwhile refuses any other push of it. + const epoch = item.session.moves ?? 0 + let opened = false + let token: string | undefined + const open = (part: AISessionBackupPush): AISessionBackupPush => { + const first = !opened + opened = true + // A push split over parts (the first says more follow) names itself on each, so + // the server keeps the session unlisted between them; a whole one opens with the + // head on whichever part goes first. + if (first && part.partial) token = randomUUID() + return { + ...part, + epoch, + ...(plan.whole ? { whole: true } : {}), + ...(plan.whole && first ? { head: plan.entry?.head } : {}), + ...(token ? { push: token } : {}), + ...(token && first ? { opens: true } : {}) + } + } + let images: AISessionBackupImage[] = [] + let imagesBytes = 0 + for (const { chat_id, id } of plan.images) { + const data_url = await readImageDataUrl(id, email) + // Evicted since the plan was made; the next save of that chat drops the id. + if (!data_url) continue + if ( + images.length > 0 && + (images.length >= MAX_IMAGES_PER_ENTRY || + imagesBytes + data_url.length > REQUEST_TARGET_BYTES) + ) { + const status = await append(open({ id: item.session.id, images, partial: true })) + if (status !== 'ok') return finish(status) + images = [] + imagesBytes = 0 + } + images.push({ chat_id, id, data_url }) + imagesBytes += data_url.length + } + // Every part but the last says so: the server lists a session on the part that + // completes its entry, never on one an unsent part still follows. + const parts = plan.entry + ? splitEntry( + plan.whole ? { ...plan.entry, head: undefined } : plan.entry, + REQUEST_TARGET_BYTES + ) + : [] + if (images.length > 0) { + const status = await append( + open({ + id: item.session.id, + images, + partial: parts.length > 0 || undefined + }) + ) + if (status !== 'ok') return finish(status) + } + for (const [i, part] of parts.entries()) { + if (failed.has(item.session.id)) break + const status = await append(open(i < parts.length - 1 ? { ...part, partial: true } : part)) + if (status !== 'ok') return finish(status) + } + attempted.get(item.session.id)!.complete = true + } + return finish(await flushCurrent()) +} + +async function flush(): Promise { + const email = getCurrentUserEmail() + if (!email) return + if (Date.now() < retryAt) { + clearTimeout(retryTimer) + retryTimer = setTimeout(runFlush, retryAt - Date.now()) + return + } + await withUserLock(email, async () => { + const marks = readPending() + // A store that cannot be opened right now (another tab's upgrade in progress) is + // retried with backoff, as any other unavailable store below. + if (!(await backfillMarks(marks, email))) { + backOff() + return + } + // Read once: retired marks are not reclaimed (a mark cannot be deleted without a + // window in which a bump is lost), so there is one per session ever backed up, and + // telling them apart from live ones is what lets a flush with nothing to do stop + // here, before the sessions store. + const syncRows = new Map((await allSyncRows(email)).map((row) => [row.id, row])) + // A mark's counter counts with the bumps its row carries (the ones localStorage refused). + const live = marks.dirty + .map((d) => ({ id: d.id, v: d.v + (syncRows.get(d.id)?.extraV ?? 0) })) + .filter((d) => { + const sync = syncRows.get(d.id) + return !(sync && !sync.stale && (sync.flushedV ?? -1) >= d.v) + }) + // A removal whose mark could not be written rides on the sync row instead. + const removals: { id: string; ws?: string; key?: string; storages?: string[] }[] = [ + ...marks.removed + ] + for (const row of syncRows.values()) { + if (row.removed && !removals.some((r) => r.id === row.id)) { + removals.push({ id: row.id, ws: row.ws }) + } + } + if (live.length === 0 && removals.length === 0) return + const stored = await readStoredSessions(email) + if (!stored) { + backOff() + return + } + const byId = new Map(stored.map((s) => [s.id, s])) + const work = new Map() + const workFor = (ws: string) => { + let w = work.get(ws) + if (!w) work.set(ws, (w = { items: [], removed: [] })) + return w + } + const droppedDirty: string[] = [] + const consumedRemoved: string[] = [] + + for (const r of removals) { + const sync = syncRows.get(r.id) + const ws = r.ws ?? sync?.ws + // Nowhere to remove it from (an unsent draft): done. A workspace whose backups are + // off keeps the removal of a session that was backed up, for when they are on + // again, or the session would come back; one never backed up from here has + // nothing there, so its mark goes, or a storage-less instance would collect one + // per deleted session forever. + if (!ws) { + if (r.key) consumedRemoved.push(r.key) + } else if (!isOff(ws) && wsState.get(ws) !== 'refused') { + // The row says where the copy is only for its own workspace: a session that + // moved on has the row its new workspace wrote, and its mark says instead. + const own = sync?.ws === ws ? sync : undefined + const holding = (own ? [own.storageId, ...(own.alsoIn ?? [])] : (r.storages ?? [])).filter( + (x): x is string => x !== undefined + ) + workFor(ws).removed.push({ + id: r.id, + key: r.key, + storageId: holding[0], + alsoIn: holding.slice(1) + }) + } else if (isOff(ws) && !sync && r.key) consumedRemoved.push(r.key) + } + for (const d of live) { + const session = byId.get(d.id) + // Gone from the store, so nothing can bump it again; an unsent draft keeps its + // mark, since it may commit to a workspace while this flush runs. + if (!session) { + droppedDirty.push(d.id) + continue + } + if (session.transient || !session.workspace_id) continue + // Left in place for a workspace that is off or refused: a move into it must still + // remember the old copy, and a mark costs one lookup per flush. + if (isOff(session.workspace_id) || wsState.get(session.workspace_id) === 'refused') { + continue + } + const sync = syncRows.get(d.id) + // A stale row plans like no row at all: the whole session goes again. One naming + // another workspace still says where the old copy is, stale or not. + workFor(session.workspace_id).items.push({ + session, + v: d.v, + sync: sync?.stale && sync.ws === session.workspace_id ? undefined : sync + }) + } + + let settledAny = false + let leftForNext = false + for (const [ws, w] of work) { + const out = await pushWorkspace(ws, w, email) + if (out.status === 'abort') return + if (out.status === 'off') { + markOff(ws) + await staleWorkspaceSync(ws, email) + // Same rule as the loop above: a removal is worth keeping only for a session + // that was backed up from here. + for (const r of w.removed) { + if (!syncRows.has(r.id) && r.key) consumedRemoved.push(r.key) + } + continue + } + // Refused stops the workspace for the page, but what the earlier requests of this + // flush stored is recorded like any other. + if (out.status === 'refused') wsState.set(ws, 'refused') + if (out.status === 'transient' || out.anyFailed || out.unavailable) backOff() + else settledAny = true + // The new workspace holds a moved session now, so the old copy can go: its removal + // mark is written before the row that forgets where the old copy was, and a + // session whose mark could not be written keeps its old row, so the next flush + // plans the move again rather than orphan the copy. + const recorded = out.settled.filter((s) => { + if (s.removeFrom) { + const old = syncRows.get(s.id) + const storages = + old?.ws === s.removeFrom + ? [old.storageId, ...(old.alsoIn ?? [])].filter((x): x is string => x !== undefined) + : undefined + if (!addRemoved(s.id, s.removeFrom, false, undefined, storages)) return false + } + if (s.removeFrom || s.carried) leftForNext = true + return true + }) + // A session with deletes carried over stays one bump short of retired, so the + // next flush sends the rest. + const written = recorded.map((s) => ({ + ...s.next, + flushedV: s.carried ? s.v - 1 : s.v, + storageId: s.storageId, + generation: s.generation + })) + await writeSync(written, email) + // The server names the storage and generation every answer comes from. Rows + // naming another go stale, the ones written just now included: a session answered + // from a storage the later answers left behind, or pushed in part on top of a row + // from another one, has its backup split across buckets the server no longer + // looks at as a whole. + if (out.storageId !== undefined) { + const storageId = out.storageId + const generation = out.generation ?? 0 + const elsewhere = (row: MirrorSyncState) => + row.storageId !== storageId || (row.generation ?? 0) !== generation + const removed = new Set(out.removedDone.map((r) => r.id)) + const writtenIds = new Set(written.map((row) => row.id)) + const untouched = [...syncRows.values()].filter( + (row) => !removed.has(row.id) && !writtenIds.has(row.id) + ) + const foreign = [ + ...foreignRows(ws, storageId, generation, untouched), + ...written.filter((row) => { + const prior = syncRows.get(row.id) + const partial = prior && !prior.stale && prior.ws === ws && elsewhere(prior) + return elsewhere(row) || partial + }) + ] + if (foreign.length > 0) { + await markStale(foreign, email) + leftForNext = true + } + } + droppedDirty.push(...out.dropped.map((d) => d.id)) + // The server holds no head of these any more (another device removed the backup): + // stale rows send them whole next. + const headless = out.needsWhole + .map((id) => syncRows.get(id)) + .filter((row): row is MirrorSyncState => row !== undefined && !row.stale) + if (headless.length > 0) { + await markStale(headless, email) + leftForNext = true + } + for (const r of out.removedDone) { + // The row describes this workspace's copy only; a session that moved on keeps + // the row its new workspace wrote. + if ((await readSync(r.id, email))?.ws === ws) await deleteSync([r.id], email) + if (r.key) consumedRemoved.push(r.key) + } + for (const r of out.removedFrom) { + const row = await readSync(r.id, email) + const [storageId, ...alsoIn] = r.remaining + if (row?.ws === ws) await writeSync([{ ...row, storageId, alsoIn }], email) + else if (r.key) { + try { + localStorage.setItem(r.key, JSON.stringify(r.remaining)) + } catch {} + } + } + } + if (settledAny && Date.now() >= retryAt) retryMs = RETRY_MIN_MS + if (getCurrentUserEmail() !== email) return + for (const id of droppedDirty) dropDirty(id) + for (const key of consumedRemoved) removeKey(key) + // Whatever is still marked is either waiting on the backoff timer, on a write that + // scheduled its own flush, or on a workspace that is off for a while or refused for the page; + // none of it wants another flush in 15 s. What this flush left for the next one + // does: a moved session's removal from its old workspace, carried-over deletes, or + // the sessions of a storage the server no longer answers from. + if (leftForNext) scheduleFlush() + }) +} + +// --- Restore --- + +function asRecord(value: unknown): Record | undefined { + return value && typeof value === 'object' && !Array.isArray(value) + ? (value as Record) + : undefined +} + +/** Turn one pulled backup into store rows, dropping anything that does not name this + * session: the bucket is written by the server from validated ids, but a record is + * still data from outside this browser. */ +function unpackBackup( + ws: string, + backup: AISessionBackup, + updatedAt: number, + storageId: string | undefined, + generation: number | undefined, + earlierChats: Iterable = [] +): + | { + session: Session + chats: StoredChat[] + images: RestoredImage[] + artifacts: { items: PersistedArtifact[]; versions: ArtifactVersion[] } + sync: MirrorSyncState + } + | undefined { + const head = asRecord(backup.head) + if (!head || head.id !== backup.id || head.workspace_id !== ws) return undefined + if (typeof head.createdAt !== 'number') return undefined + const chats: StoredChat[] = [] + for (const c of backup.chats) { + const record = asRecord(c.record) + if ( + !record || + record.id !== c.id || + record.sessionId !== backup.id || + !Array.isArray(record.actualMessages) || + !Array.isArray(record.displayMessages) || + typeof record.lastModified !== 'number' + ) { + continue + } + chats.push(record as unknown as StoredChat) + } + // An image's chat may have come on an earlier page of the session. + const chatIds = new Set([...chats.map((c) => c.id), ...earlierChats]) + const images: RestoredImage[] = backup.images + .filter((i) => chatIds.has(i.chat_id) && typeof i.data_url === 'string') + .map((i) => ({ id: i.id, chatId: i.chat_id, dataUrl: i.data_url })) + const artifactsRecord = asRecord(backup.artifacts) + const items = ( + Array.isArray(artifactsRecord?.items) ? (artifactsRecord.items as PersistedArtifact[]) : [] + ).filter((i) => asRecord(i)?.sessionId === backup.id && typeof i.id === 'string') + const itemIds = new Set(items.map((i) => i.id)) + const versions = ( + Array.isArray(artifactsRecord?.versions) ? (artifactsRecord.versions as ArtifactVersion[]) : [] + ).filter((v) => asRecord(v) && itemIds.has(v.artifactId) && typeof v.key === 'string') + const active = chats.find((c) => c.id === head.chatId) + const session: Session = { + ...(head as unknown as Session), + name: '', + // Everything in the backup has been read here: no unread badge, and the last + // activity is the backup's own time. + lastSeenCount: active?.displayMessages.length ?? 0, + lastActivityAt: updatedAt + } + const sync: MirrorSyncState = { + id: backup.id, + ws, + head: headSig(session), + chats: Object.fromEntries(chats.map((c) => [c.id, c.lastModified])), + images: Object.fromEntries(images.map((i) => [i.id, i.chatId])), + artifacts: artifactsFingerprint({ items, versions }), + storageId, + generation + } + return { session, chats, images, artifacts: { items, versions }, sync } +} + +type BackupListing = Awaited> + +/** The workspace's listing; `off` when it keeps no backups, `failed` when it could not be + * listed this time. */ +async function listWorkspace(ws: string, email: string): Promise { + let listing + try { + listing = await AiService.listAiSessionBackups({ workspace: ws }) + } catch (e) { + const status = statusOf(e) + if (status === 404 || status === 403) { + markOff(ws) + return 'off' + } + console.warn('Could not list session backups', e) + return 'failed' + } + if (!listing.enabled) { + markOff(ws) + return 'off' + } + if (wsState.get(ws) !== 'refused') wsState.set(ws, 'on') + const foreign = + listing.storage_id === undefined + ? [] + : foreignRows( + ws, + listing.storage_id, + listing.backup_generation ?? 0, + await allSyncRows(email) + ) + if (foreign.length > 0) { + await markStale(foreign, email) + scheduleFlush() + } + return listing +} + +/** The workspaces of one family are restored together: a session moved between two of them + * is listed by both until the old copy's removal lands (that mark is the moving browser's, + * which may never come back), and whichever were restored first would take the id and keep + * the other out. The copy that moved last (`epoch`, the record's move count, kept with the + * marker) is the one brought back, the storage's own modification time deciding between two + * of the same count; the other is left where it is. A family one of whose workspaces could + * not be listed is not restored at all this time, or the copy that lists could be the stale + * one; the next page load or workspace switch tries again. */ +async function restoreFamily(family: string[], todo: string[], email: string): Promise { + const listings = new Map() + for (const ws of todo) { + if (getCurrentUserEmail() !== email) return + const listing = await listWorkspace(ws, email) + if (listing === 'failed') { + for (const w of todo) restoredWorkspaces.delete(w) + return + } + if (listing !== 'off') listings.set(ws, listing) + // Probed again once its backups may be on again (see `isOff`). + else restoredWorkspaces.delete(ws) + } + const newest = new Map() + for (const [ws, listing] of listings) { + for (const s of listing.sessions) { + const at = Date.parse(s.updated_at) + const cur = newest.get(s.id) + if (!cur || s.epoch > cur.epoch || (s.epoch === cur.epoch && at > cur.at)) { + newest.set(s.id, { ws, epoch: s.epoch, at }) + } + } + } + // A move landing between the listings and the pulls (another device pushing the session + // into a workspace listed before it held it, or one whose backups were off then) would + // make a copy about to be imported the stale one: the whole family is listed again once + // a workspace's pulls are done and its records are about to land, and a session a later + // copy of which showed up elsewhere is left, with the family, for next time. A family of + // one has nowhere else for a copy to show up. + const verify = async (ids: string[]): Promise> => { + const superseded = new Set() + if (family.length < 2) return superseded + for (const w of family) { + const listing = await listWorkspace(w, email) + if (listing === 'failed') { + for (const id of ids) superseded.add(id) + break + } + if (listing === 'off') continue + for (const s of listing.sessions) { + const cur = newest.get(s.id) + if (!cur || !ids.includes(s.id) || w === cur.ws) continue + const at = Date.parse(s.updated_at) + if (s.epoch > cur.epoch || (s.epoch === cur.epoch && at > cur.at)) superseded.add(s.id) + } + } + if (superseded.size > 0) for (const w of todo) restoredWorkspaces.delete(w) + return superseded + } + for (const [ws, listing] of listings) { + if (getCurrentUserEmail() !== email) return + const elsewhere = listing.sessions.filter((s) => newest.get(s.id)?.ws !== ws).map((s) => s.id) + await restoreWorkspace(ws, email, listing, new Set(elsewhere), verify) + } +} + +async function restoreWorkspace( + ws: string, + email: string, + listing: BackupListing, + elsewhere: Set, + verify: (ids: string[]) => Promise> +): Promise { + const rows = await allSyncRows(email) + const local = new Set() + for (const s of (await readStoredSessions(email)) ?? []) local.add(s.id) + for (const s of sessionState.sessions) local.add(s.id) + // A removal names the workspace it is for: a session that moved here from another one + // still has that one's removal pending, and is ours to restore. + for (const r of readPending().removed) if (r.ws === ws) local.add(r.id) + for (const row of rows) if (row.removed && row.ws === ws) local.add(row.id) + const candidates = listing.sessions + .filter((s) => !local.has(s.id) && !elsewhere.has(s.id) && !isSessionTombstoned(s.id)) + .slice(0, RESTORE_MAX) + const updatedAt = new Map(candidates.map((s) => [s.id, Date.parse(s.updated_at)])) + let ids = candidates.map((s) => s.id) + const toImport: Staged[] = [] + // A session that did not fit one answer whole comes in pages, kept here until the last + // one: importing a page alone would leave a session the next restore takes for whole. + type Pieces = { + chats: Set + images: Set + items: Set + versions: Set + } + type Staged = { + session: Session + sync: MirrorSyncState + /** Everything the pages so far wrote for the session. */ + pieces: Pieces + /** The listing fingerprint the pages so far were answered with. */ + listing?: string + } + const staged = new Map() + const noPieces = (): Pieces => ({ + chats: new Set(), + images: new Set(), + items: new Set(), + versions: new Set() + }) + const union = (a: Pieces, b: Pieces): Pieces => ({ + chats: new Set([...a.chats, ...b.chats]), + images: new Set([...a.images, ...b.images]), + items: new Set([...a.items, ...b.items]), + versions: new Set([...a.versions, ...b.versions]) + }) + // What earlier attempts (a restore cut short, a start over) wrote for a session that has + // no record yet, from their staging rows: the pieces of it the backup no longer has go + // before the record lands, or a later flush would push them back. Ids, never clocks. + const earlierStaging = new Map() + for (const row of rows) { + if (row.staging) { + earlierStaging.set(row.id, { + chats: new Set(row.staging.chats), + images: new Set(row.staging.images), + items: new Set(row.staging.items), + versions: new Set(row.staging.versions) + }) + } + } + // A session whose backup moved between two of its pages starts over, a few times. + const restarts = new Map() + const MAX_RESTARTS = 3 + const resumes: AISessionBackupCursor[] = [] + while (ids.length > 0 || resumes.length > 0) { + if (getCurrentUserEmail() !== email) return + const resume = resumes.shift() + const batch = resume ? [resume.id] : ids.slice(0, PULL_BATCH) + if (!resume) ids = ids.slice(PULL_BATCH) + let pulled + try { + pulled = await AiService.pullAiSessionBackups({ + workspace: ws, + requestBody: { ids: batch, resume } + }) + } catch (e) { + console.warn('Could not pull session backups', e) + return + } + if (!pulled.enabled) { + markOff(ws) + return + } + // Ask again for what did not fit, one at a time so each answer is as small as can be. + for (const id of pulled.deferred) if (!ids.includes(id)) ids.unshift(id) + // A session's pieces land before its record, page by page (the writes are absent-only, + // so a restore cut short leaves nothing a later one cannot finish), and the record, + // which is what makes the session visible, only with the last page; a session whose + // pieces could not be written is left for the next restore, since recording it now + // would let the next flush push its half-empty local state over the backup. What a + // page leaves for the next is the sync row being assembled, never its pieces. + const ready: Staged[] = [] + for (const b of pulled.sessions) { + const earlier = staged.get(b.id) + staged.delete(b.id) + // Brought back meanwhile by this tab itself (a session moved here from another + // workspace, say; the lock keeps other tabs out): its pieces are not ours to write + // over any more. + if ((await readStoredSessions(email))?.some((s) => s.id === b.id)) continue + // The backup moved between two pages (a chat sorting before the cursor would be + // missed) or under this one (the page may mix two versions): the pages so far do + // not belong together, the session starts over. + if (b.moved || (earlier && b.listing !== earlier.listing)) { + const n = (restarts.get(b.id) ?? 0) + 1 + restarts.set(b.id, n) + if (earlier) { + earlierStaging.set(b.id, union(earlierStaging.get(b.id) ?? noPieces(), earlier.pieces)) + } + if (n < MAX_RESTARTS) ids.unshift(b.id) + else console.warn(`Session backup ${b.id} kept changing while restoring; left for later`) + continue + } + const u = unpackBackup( + ws, + b, + updatedAt.get(b.id) ?? Date.now(), + pulled.storage_id, + pulled.backup_generation, + Object.keys(earlier?.sync.chats ?? {}) + ) + if (!u) continue + const written: Pieces = { + chats: new Set(u.chats.map((c) => c.id)), + images: new Set(u.images.map((i) => i.id)), + items: new Set(u.artifacts.items.map((i) => i.id)), + versions: new Set(u.artifacts.versions.map((v) => v.key)) + } + const merged: Staged = earlier + ? { + session: { + ...u.session, + lastSeenCount: Math.max( + earlier.session.lastSeenCount ?? 0, + u.session.lastSeenCount ?? 0 + ) + }, + sync: { + ...u.sync, + chats: { ...earlier.sync.chats, ...u.sync.chats }, + images: { ...earlier.sync.images, ...u.sync.images }, + artifacts: b.artifacts !== undefined ? u.sync.artifacts : earlier.sync.artifacts + }, + pieces: union(earlier.pieces, written) + } + : { session: u.session, sync: u.sync, pieces: written } + merged.listing = b.listing + // The staging row goes before the pieces, and outlives a restore cut short: the + // next one reads it to know what to delete. The record replaces it. + const stagingPieces = union(earlierStaging.get(b.id) ?? noPieces(), merged.pieces) + await writeSync( + [ + { + id: b.id, + ws, + head: '', + chats: {}, + images: {}, + staging: { + chats: [...stagingPieces.chats], + images: [...stagingPieces.images], + items: [...stagingPieces.items], + versions: [...stagingPieces.versions] + } + } + ], + email + ) + // Written over whatever is there: the session is absent locally, so its pieces + // can only be what an earlier restore staged before it was cut short, and the + // backup may have moved on since. + try { + if (!(await importArtifacts(u.artifacts.items, u.artifacts.versions, email, true))) continue + if (!(await importStoredChats(u.chats, u.images, email, true))) continue + } catch (e) { + console.error(`Could not restore session ${u.session.id}`, e) + continue + } + if (b.next) { + staged.set(b.id, merged) + resumes.push(b.next) + continue + } + ready.push(merged) + } + if (ready.length === 0) continue + // The staged pieces the backup no longer has go before the record: once the record + // is there, no restore looks at the session again, and a flush would push them back. + // A prune that could not run leaves the session, its pieces and its staging row for + // the next restore. + const pruned: Staged[] = [] + for (const r of ready) { + const prior = earlierStaging.get(r.session.id) + if (prior) { + const gone = (was: Set, now: Set) => + new Set([...was].filter((id) => !now.has(id))) + const ok = + (await pruneSessionChats( + r.session.id, + gone(prior.chats, r.pieces.chats), + gone(prior.images, r.pieces.images), + email + )) && + (await pruneSessionArtifacts( + r.session.id, + gone(prior.items, r.pieces.items), + gone(prior.versions, r.pieces.versions), + email + )) + if (!ok) { + console.warn(`Session backup ${r.session.id} could not be tidied; left for later`) + continue + } + earlierStaging.delete(r.session.id) + } + pruned.push(r) + } + toImport.push(...pruned) + } + // The records land together once every pull is done, so the family is listed again + // once per workspace rather than per answer; the pieces are in place either way, and a + // restore cut short before this leaves them staged for the next. + if (toImport.length === 0) return + const superseded = await verify(toImport.map((r) => r.session.id)) + const current = toImport.filter((r) => !superseded.has(r.session.id)) + if (current.length === 0) return + const imported = new Set( + await importSessions( + current.map((r) => r.session), + email + ) + ) + await writeSync( + current.filter((r) => imported.has(r.session.id)).map((r) => r.sync), + email + ) + if (imported.size > 0) logFeatureUsage('ai_session', 'restored', { value: imported.size }) +} + +/** + * Restore the sessions of the workspace family the user is looking at (the workspace and + * its forks), once per workspace per page load. Sessions that exist here are never + * touched; only ones this browser lacks are brought back. + */ +export function restoreSessionBackups(currentWorkspace: string): void { + if (!BROWSER) return + // A restore writes an absent session's pieces page by page and prunes what an earlier + // one staged: two tabs doing that at once would write over each other, so it runs only + // under the tab lock. Where Web Locks do not exist (a plain http origin), the browser + // still backs up; its sessions come back on a secure one. + if (!hasWebLocks()) return + const email = getCurrentUserEmail() + if (!email) return + const all = get(userWorkspaces) + const root = workspaceRootId(currentWorkspace, all) ?? currentWorkspace + const family = new Set([currentWorkspace]) + for (const w of all) if ((workspaceRootId(w.id, all) ?? w.id) === root) family.add(w.id) + const todo = [...family].filter((ws) => !restoredWorkspaces.has(ws) && !isOff(ws)) + for (const ws of todo) restoredWorkspaces.add(ws) + if (todo.length > 0) { + void enqueue(() => withUserLock(email, () => restoreFamily([...family], todo, email), true)) + } +} + +/** The workspace's backups were just turned on or off from this page: what was learnt of + * them is forgotten, and the next flush and a restore find out afresh. */ +export function backupSettingsChanged(ws: string): void { + forgetOff(ws) + wsState.delete(ws) + restoredWorkspaces.delete(ws) + // The rows went stale when the backups went off: the sessions are marked again. + backfilled = false + scheduleFlush() + restoreSessionBackups(ws) +} + +// --- Wiring --- + +if (BROWSER) { + onMirrorSignal((signal) => { + // A mark for another user waits for that user's next load. + const mine = !signal.email || signal.email === getCurrentUserEmail() + if (signal.kind === 'dirty') { + if (bumpDirty(signal.sessionId, signal.email)) { + if (mine) scheduleFlush() + } else { + if (mine && !unwritableMarks.has(signal.sessionId)) { + unwritableMarks.set(signal.sessionId, 0) + } + void bumpViaSyncRow(signal.sessionId, signal.email).finally(() => { + if (mine) scheduleFlush() + }) + } + return + } + if (!addRemoved(signal.sessionId, signal.workspaceId, true, signal.email)) { + void removeViaSyncRow(signal.sessionId, signal.workspaceId, signal.email) + } + if (mine) scheduleFlush() + }) + onUserChange((email) => { + clearTimers() + clearTimeout(retryTimer) + retryAt = 0 + retryMs = RETRY_MIN_MS + wsState.clear() + clearOffTimers() + restoredWorkspaces.clear() + backfilled = false + unwritableMarks.clear() + if (email) setTimeout(runFlush, STARTUP_DELAY_MS) + }) + // A tab going to the background may not come back: carry what it has now. + if (typeof document !== 'undefined') { + document.addEventListener('visibilitychange', () => { + if (document.visibilityState === 'hidden') runFlush() + }) + } +} + +/** Test-only: run a flush now, outside the timers. */ +export function __flushForTesting(): Promise { + return enqueue(flush) +} + +/** Test-only: what the sync table holds for the user. */ +export async function __syncRowsForTesting(email: string): Promise { + return allSyncRows(email) +} + +/** Test-only: how long a workspace found off is left alone before the page asks again. */ +export function __setOffRetryForTesting(ms: number): void { + offRetryMs = ms +} + +/** Test-only: plant sync rows, as an earlier page load would have left them. */ +export function __writeSyncForTesting(rows: MirrorSyncState[], email: string): Promise { + return writeSync(rows, email) +} + +/** Test-only: wait for whatever flush or restore is queued. */ +export function __settleForTesting(): Promise { + return enqueue(async () => {}) +} + +/** Test-only: forget every page-lifetime decision, and let go of the sync store so the + * next open lands in the test's fresh IndexedDB rather than the cached connection. */ +export function __resetMirrorForTesting(): void { + syncDbh.close() + clearTimers() + clearTimeout(retryTimer) + retryAt = 0 + retryMs = RETRY_MIN_MS + wsState.clear() + clearOffTimers() + offRetryMs = OFF_RETRY_MS + restoredWorkspaces.clear() + backfilled = false + unwritableMarks.clear() +} diff --git a/frontend/src/lib/components/sessions/sessionMirror.test.ts b/frontend/src/lib/components/sessions/sessionMirror.test.ts new file mode 100644 index 0000000000..d7260c0878 --- /dev/null +++ b/frontend/src/lib/components/sessions/sessionMirror.test.ts @@ -0,0 +1,1449 @@ +import { describe, it, expect, beforeEach, vi } from 'vitest' +import { IDBFactory } from 'fake-indexeddb' + +// The stores are BROWSER-gated; the vitest "server" env reports false. +vi.mock('esm-env', async (importOriginal) => ({ + ...(await importOriginal()), + BROWSER: true +})) + +const { pushMock, listMock, pullMock } = vi.hoisted(() => ({ + pushMock: vi.fn(), + listMock: vi.fn(), + pullMock: vi.fn() +})) +vi.mock('$lib/gen', async (orig) => { + const actual = await orig() + return { + ...actual, + AiService: { + ...actual.AiService, + pushAiSessionBackups: pushMock, + listAiSessionBackups: listMock, + pullAiSessionBackups: pullMock + }, + WorkspaceService: { + ...actual.WorkspaceService, + listUserWorkspaces: vi.fn().mockResolvedValue([]), + getSessionWorkspaceStatus: vi.fn().mockResolvedValue({}) + } + } +}) + +// The chat store during a restore: real, or unreachable for one session. +const { chatImport } = vi.hoisted(() => ({ + chatImport: { unavailable: false, pruneFails: false } +})) +vi.mock('../copilot/chat/HistoryManager.svelte', async (orig) => { + const actual = await orig() + return { + ...actual, + importStoredChats: (...args: Parameters) => + chatImport.unavailable ? Promise.resolve(false) : actual.importStoredChats(...args), + pruneSessionChats: (...args: Parameters) => + chatImport.pruneFails ? Promise.resolve(false) : actual.pruneSessionChats(...args) + } +}) + +/** The Web Locks API, which the node test environment lacks: one holder per name at a + * time, `ifAvailable` answering null while a holder is there. */ +function fakeLockManager(): LockManager { + const tails = new Map>() + return { + request: async ( + name: string, + options: LockOptions | undefined, + cb: (lock: Lock | null) => unknown + ) => { + const prev = tails.get(name) + if (options?.ifAvailable && prev) return cb(null) + const run = (prev ?? Promise.resolve()).then(() => cb({ name, mode: 'exclusive' })) + const tail = run.catch(() => {}) + tails.set(name, tail) + try { + return await run + } finally { + if (tails.get(name) === tail) tails.delete(name) + } + } + } as unknown as LockManager +} + +function setWebLocks(locks: LockManager | undefined): void { + if (typeof navigator === 'undefined') { + Object.defineProperty(globalThis, 'navigator', { + value: {}, + configurable: true, + writable: true + }) + } + Object.defineProperty(navigator, 'locks', { value: locks, configurable: true }) +} + +import { superadmin, userStore, usersWorkspaceStore, type UserExt } from '$lib/stores' +import HistoryManager, { + __resetBackupStoreForTesting, + __resetLegacyChatClaimForTesting, + readStoredChat +} from '../copilot/chat/HistoryManager.svelte' +import { + deleteSession, + importSessions, + putSession, + sessionState, + type Session +} from './sessionState.svelte' +import { markSessionDirty } from './sessionMirrorSignal' +import { + __flushForTesting, + __resetMirrorForTesting, + __settleForTesting, + __setOffRetryForTesting, + __syncRowsForTesting, + __writeSyncForTesting, + backupSettingsChanged, + restoreSessionBackups +} from './sessionMirror.svelte' + +const EMAIL = 'mirror@x.com' +const IMAGE = 'data:image/png;base64,AAAA' +const PENDING_PREFIX = `windmill_sessions_mirror_pending::${EMAIL}::` + +function asUser(email: string): UserExt { + return { email, username: email.split('@')[0] } as unknown as UserExt +} +const flush = () => new Promise((r) => setTimeout(r, 0)) + +function pendingKeys(): string[] { + const keys: string[] = [] + for (let i = 0; i < localStorage.length; i++) { + const key = localStorage.key(i) + if (key?.startsWith(PENDING_PREFIX)) keys.push(key.slice(PENDING_PREFIX.length)) + } + return keys.sort() +} + +function removalKeys(): string[] { + return pendingKeys().filter((k) => k.startsWith('r::')) +} + +/** Sessions whose dirty counter no push has covered yet. */ +async function pendingDirty(): Promise { + const flushed = new Map() + for (const r of await __syncRowsForTesting(EMAIL)) { + flushed.set(r.id, r.stale ? -1 : (r.flushedV ?? -1) - (r.extraV ?? 0)) + } + const out: string[] = [] + for (const key of pendingKeys()) { + if (!key.startsWith('d::')) continue + const id = key.slice(3) + if (Number(localStorage.getItem(PENDING_PREFIX + key)) > (flushed.get(id) ?? -1)) out.push(id) + } + return out.sort() +} + +beforeEach(async () => { + ;(globalThis as any).indexedDB = new IDBFactory() + localStorage.clear() + __resetLegacyChatClaimForTesting() + __resetBackupStoreForTesting() + __resetMirrorForTesting() + pushMock.mockReset() + listMock.mockReset() + pullMock.mockReset() + chatImport.unavailable = false + chatImport.pruneFails = false + setWebLocks(fakeLockManager()) + superadmin.set(false) + usersWorkspaceStore.set(undefined) + userStore.set(undefined) + await flush() + sessionState.sessions = [] + userStore.set(asUser(EMAIL)) + await vi.waitFor(() => expect(sessionState.hydrated).toBe(true)) +}) + +// The whole loop, end to end against the real stores: local writes mark, a flush sends +// one batch carrying the record, the chat and its image, reading the session sends +// nothing, and a user delete removes the backup. +describe('sessionMirror flush', () => { + it('pushes what changed, once, and removes a deleted session', async () => { + pushMock.mockResolvedValue({ enabled: true, results: [{ id: 's1' }] }) + const s: Session = { + id: 's1', + name: 'session-1', + createdAt: 1, + workspace_id: 'ws', + chatId: 'c1' + } + sessionState.sessions = [s] + await putSession(s) + + const hm = new HistoryManager() + await hm.init() + hm.setSessionId('s1') + hm.setCurrentChatId('c1') + await hm.saveChat( + [{ role: 'user', content: 'hello', images: [{ dataUrl: IMAGE, name: 'a.png' }] } as never], + [{ role: 'user', content: [{ type: 'image_url', image_url: { url: IMAGE } }] } as never] + ) + + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(1) + const body = pushMock.mock.calls[0][0].requestBody + expect(pushMock.mock.calls[0][0].workspace).toBe('ws') + expect(body.owner).toBe(EMAIL) + expect(body.removed).toBeUndefined() + const [imageEntry, entry] = body.sessions + expect(imageEntry.images).toEqual([{ chat_id: 'c1', id: expect.any(String), data_url: IMAGE }]) + // A first push goes whole, under one token on every part: the head opens it on the + // first part, whichever that is. + expect(imageEntry.whole).toBe(true) + expect(imageEntry.partial).toBe(true) + expect(typeof imageEntry.push).toBe('string') + expect(imageEntry.opens).toBe(true) + expect(imageEntry.head).toEqual({ id: 's1', createdAt: 1, workspace_id: 'ws', chatId: 'c1' }) + expect(entry.head).toBeUndefined() + expect(entry.whole).toBe(true) + expect(entry.push).toBe(imageEntry.push) + expect(entry.opens).toBeUndefined() + expect(entry.chats.map((c: { id: string }) => c.id)).toEqual(['c1']) + // The record keeps its blob ref; bytes travel as the image object only. + expect(JSON.stringify(entry.chats[0].record)).not.toContain(IMAGE) + expect(entry.artifacts).toBeUndefined() + expect(await pendingDirty()).toEqual([]) + + // Reading the session is not a change the backup keeps. + await putSession({ ...s, lastSeenCount: 2, lastActivityAt: 99 }) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(1) + + // A user delete takes the backup with it. + deleteSession('s1') + await flush() + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(2) + expect(pushMock.mock.calls[1][0].requestBody).toEqual({ + owner: EMAIL, + sessions: [], + removed: ['s1'] + }) + expect(pendingKeys()).toEqual([]) + hm.close() + }) + + it('carries a user delete on the sync row when its localStorage mark cannot be written', async () => { + pushMock.mockResolvedValue({ enabled: true, results: [{ id: 'sd' }] }) + const s: Session = { id: 'sd', name: 'session-1', createdAt: 1, workspace_id: 'ws' } + sessionState.sessions = [s] + await putSession(s) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(1) + + // Storage full at the moment of the delete. + const setItem = localStorage.setItem.bind(localStorage) + localStorage.setItem = (key: string, value: string) => { + if (key.includes('::r::')) throw new Error('QuotaExceededError') + setItem(key, value) + } + try { + deleteSession('sd') + } finally { + localStorage.setItem = setItem + } + await flush() + expect(removalKeys()).toEqual([]) + await vi.waitFor(async () => + expect((await __syncRowsForTesting(EMAIL)).find((r) => r.id === 'sd')?.removed).toBe(true) + ) + + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(2) + expect(pushMock.mock.calls[1][0].requestBody).toEqual({ + owner: EMAIL, + sessions: [], + removed: ['sd'] + }) + expect((await __syncRowsForTesting(EMAIL)).some((r) => r.id === 'sd')).toBe(false) + await __settleForTesting() + }) + + it('keeps a delete filed on the sync row while the first push is still in flight', async () => { + const s: Session = { id: 'sr', name: 'session-1', createdAt: 1, workspace_id: 'ws' } + sessionState.sessions = [s] + await putSession(s) + let release!: (value: unknown) => void + pushMock.mockImplementationOnce(() => new Promise((r) => (release = r))) + const inFlight = __flushForTesting() + await vi.waitFor(() => expect(pushMock).toHaveBeenCalledTimes(1)) + + // Storage full at the moment of the delete, the push not yet answered. + const setItem = localStorage.setItem.bind(localStorage) + localStorage.setItem = (key: string, value: string) => { + if (key.includes('::r::')) throw new Error('QuotaExceededError') + setItem(key, value) + } + try { + deleteSession('sr') + } finally { + localStorage.setItem = setItem + } + await vi.waitFor(async () => + expect((await __syncRowsForTesting(EMAIL)).find((r) => r.id === 'sr')?.removed).toBe(true) + ) + release({ enabled: true, results: [{ id: 'sr' }] }) + await inFlight + // The push's own row write did not lose the removal. + expect((await __syncRowsForTesting(EMAIL)).find((r) => r.id === 'sr')?.removed).toBe(true) + + pushMock.mockResolvedValueOnce({ enabled: true, results: [{ id: 'sr' }] }) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(2) + expect(pushMock.mock.calls[1][0].requestBody.removed).toEqual(['sr']) + expect((await __syncRowsForTesting(EMAIL)).some((r) => r.id === 'sr')).toBe(false) + }) + + it('keeps the marks when the push fails, and stops for a workspace without storage', async () => { + const s: Session = { id: 's2', name: 'session-2', createdAt: 1, workspace_id: 'ws' } + const never: Session = { id: 's2b', name: 'session-3', createdAt: 2, workspace_id: 'ws' } + sessionState.sessions = [s, never] + await putSession(s) + + pushMock.mockRejectedValueOnce(new TypeError('network')) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(1) + expect(await pendingDirty()).toEqual(['s2']) + // Still marked: the retry carries it again once the backoff lapses. + __resetMirrorForTesting() + pushMock.mockResolvedValueOnce({ enabled: true, results: [{ id: 's2' }] }) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(2) + expect(await pendingDirty()).toEqual([]) + + // The storage goes away: no further request for the page. A delete keeps its + // removal mark only for a session that was backed up (s2), for when backups are on + // again, or it would come back from the bucket; one never backed up has nothing + // there, so its mark goes rather than piling up on a storage-less instance. + await putSession({ ...s, summary: 'changed' }) + await putSession(never) + __setOffRetryForTesting(50) + pushMock.mockResolvedValueOnce({ enabled: false, results: [] }) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(3) + // The reconcile that runs after login may have re-read the list from the store + // before `never` was in it; deleteSession only acts on sessions it can see. + sessionState.sessions = [s, never] + deleteSession('s2') + deleteSession('s2b') + await flush() + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(3) + expect(pendingKeys()).toEqual(['r::s2::ws']) + + // Backups an admin turns on again elsewhere are noticed once the page's memory of + // them being off expires, with nothing else prompting it: the removal goes then. + listMock.mockResolvedValue({ enabled: true, sessions: [] }) + pushMock.mockResolvedValueOnce({ enabled: true, results: [{ id: 's2' }] }) + await vi.waitFor(() => expect(pushMock).toHaveBeenCalledTimes(4), { timeout: 3000 }) + expect(pushMock.mock.calls[3][0].requestBody.removed).toEqual(['s2']) + await vi.waitFor(() => expect(listMock).toHaveBeenCalledTimes(1)) + await __settleForTesting() + expect(pendingKeys()).toEqual([]) + }) + + it('backs up after a reload an edit whose bump only the sync row carries', async () => { + // Storage full from the session's first mark on: the first push carries the mark this + // page kept, and the edit landing while it is in flight goes onto the row it writes. + const s: Session = { id: 'sx', name: 'session-1', createdAt: 1, workspace_id: 'ws' } + const setItem = localStorage.setItem.bind(localStorage) + localStorage.setItem = (key: string, value: string) => { + if (key.includes('::d::')) throw new Error('QuotaExceededError') + setItem(key, value) + } + try { + sessionState.sessions = [s] + await putSession(s) + let release!: (value: unknown) => void + pushMock.mockImplementationOnce(() => new Promise((r) => (release = r))) + const inFlight = __flushForTesting() + await vi.waitFor(() => expect(pushMock).toHaveBeenCalledTimes(1)) + await putSession({ ...s, summary: 'second' }) + release({ enabled: true, results: [{ id: 'sx' }] }) + await inFlight + expect(pendingKeys()).toEqual([]) + const row = (await __syncRowsForTesting(EMAIL)).find((r) => r.id === 'sx') + expect((row?.extraV ?? 0) > (row?.flushedV ?? -1)).toBe(true) + + // A reload, storage still full: nothing in localStorage or memory names the + // session, the row does. + __resetMirrorForTesting() + pushMock.mockResolvedValueOnce({ enabled: true, results: [{ id: 'sx' }] }) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(2) + expect(pushMock.mock.calls[1][0].requestBody.sessions[0].head.summary).toBe('second') + expect(await pendingDirty()).toEqual([]) + + // The next refused bump counts from what that push retired (the row carries it; + // `pendingDirty` reads localStorage marks only, so the push is the check). + await putSession({ ...s, summary: 'third' }) + await vi.waitFor(async () => + expect((await __syncRowsForTesting(EMAIL)).find((r) => r.id === 'sx')?.extraV).toBe(3) + ) + pushMock.mockResolvedValueOnce({ enabled: true, results: [{ id: 'sx' }] }) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(3) + expect(pushMock.mock.calls[2][0].requestBody.sessions[0].head.summary).toBe('third') + expect(await pendingDirty()).toEqual([]) + } finally { + localStorage.setItem = setItem + } + }) + + it('backs up and restores again at once when the backups are turned on from this page', async () => { + const s: Session = { id: 'so', name: 'session-1', createdAt: 1, workspace_id: 'ws' } + sessionState.sessions = [s] + await putSession(s) + pushMock.mockResolvedValueOnce({ enabled: true, results: [{ id: 'so' }] }) + await __flushForTesting() + await putSession({ ...s, summary: 'changed' }) + pushMock.mockResolvedValueOnce({ enabled: false, results: [] }) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(2) + await putSession({ ...s, summary: 'changed again' }) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(2) + + listMock.mockResolvedValue({ enabled: true, sessions: [] }) + pushMock.mockResolvedValueOnce({ enabled: true, results: [{ id: 'so' }] }) + usersWorkspaceStore.set({ email: EMAIL, workspaces: [] } as never) + backupSettingsChanged('ws') + await __settleForTesting() + await __flushForTesting() + expect(listMock).toHaveBeenCalledTimes(1) + expect(pushMock).toHaveBeenCalledTimes(3) + // The rows went stale while the backups were off: the session goes whole. + expect(pushMock.mock.calls[2][0].requestBody.sessions[0].whole).toBe(true) + expect(pushMock.mock.calls[2][0].requestBody.sessions[0].head.summary).toBe('changed again') + expect(await pendingDirty()).toEqual([]) + }) + + it('settles nothing of a request that failed, even a session whose parts were still to come', async () => { + // Enough sessions for two requests: the first fails, the second is never sent. + const ids = Array.from({ length: 101 }, (_, i) => `m${i}`) + for (const id of ids) { + await putSession({ id, name: id, createdAt: 1, workspace_id: 'ws' }) + } + pushMock.mockRejectedValueOnce(new TypeError('network')) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(1) + expect(pushMock.mock.calls[0][0].requestBody.sessions).toHaveLength(100) + expect(await pendingDirty()).toHaveLength(101) + + // Once the backoff lapses, every one of them is carried again. + __resetMirrorForTesting() + pushMock.mockImplementation( + async ({ requestBody }: { requestBody: { sessions: { id: string }[] } }) => ({ + enabled: true, + results: requestBody.sessions.map((s) => ({ id: s.id })) + }) + ) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(3) + expect( + pushMock.mock.calls + .slice(1) + .flatMap((c) => c[0].requestBody.sessions.map((s: { id: string }) => s.id)) + .sort() + ).toEqual([...ids].sort()) + expect(await pendingDirty()).toEqual([]) + }) + + it('backs off when the server could not store a session, keeping its mark', async () => { + const s: Session = { id: 's3', name: 'session-3', createdAt: 1, workspace_id: 'ws' } + sessionState.sessions = [s] + await putSession(s) + + pushMock.mockResolvedValue({ enabled: true, results: [{ id: 's3', error: 'bucket refused' }] }) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(1) + // Still marked, but not re-sent until the backoff lapses. + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(1) + expect(await pendingDirty()).toEqual(['s3']) + }) + + it('fails only the sessions of a request the server found too large', async () => { + const ids = Array.from({ length: 101 }, (_, i) => `t${i}`) + for (const id of ids) { + await putSession({ id, name: id, createdAt: 1, workspace_id: 'ws' }) + } + const { ApiError } = await import('$lib/gen') + pushMock + .mockRejectedValueOnce( + new ApiError({ method: 'POST', url: '' } as never, { status: 413 } as never, 'too large') + ) + .mockImplementation( + async ({ requestBody }: { requestBody: { sessions: { id: string }[] } }) => ({ + enabled: true, + results: requestBody.sessions.map((s) => ({ id: s.id })) + }) + ) + await __flushForTesting() + // The second request still went out and settled its session; the first's stay marked. + expect(pushMock).toHaveBeenCalledTimes(2) + expect(await pendingDirty()).toHaveLength(100) + const settled = pushMock.mock.calls[1][0].requestBody.sessions[0].id + expect(await pendingDirty()).not.toContain(settled) + }) + + it('moves a session whole into its new workspace and files the old copy for removal', async () => { + const s: Session = { id: 'mv', name: 'session-1', createdAt: 1, workspace_id: 'ws' } + sessionState.sessions = [s] + await putSession(s) + pushMock.mockResolvedValueOnce({ enabled: true, results: [{ id: 'mv' }] }) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(1) + + // The old workspace's backups go off before the move: its rows go stale, but they + // still say where the copy is. + await putSession({ ...s, summary: 'changed' }) + pushMock.mockResolvedValueOnce({ enabled: false, results: [] }) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(2) + + await putSession({ ...s, summary: 'changed', workspace_id: 'ws2' }) + pushMock.mockResolvedValueOnce({ enabled: true, results: [{ id: 'mv' }] }) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(3) + expect(pushMock.mock.calls[2][0].workspace).toBe('ws2') + expect(pushMock.mock.calls[2][0].requestBody.sessions[0].head.workspace_id).toBe('ws2') + // The removal waits for the old workspace's backups to be on again. + expect(removalKeys()).toEqual(['r::mv::ws']) + expect(await pendingDirty()).toEqual([]) + }) + + it('pushes every session whole again once the server answers from another storage', async () => { + const a: Session = { + id: 'sa', + name: 'session-1', + createdAt: 1, + workspace_id: 'ws', + chatId: 'ca' + } + const b: Session = { id: 'sb', name: 'session-2', createdAt: 2, workspace_id: 'ws' } + sessionState.sessions = [a, b] + await putSession(a) + await putSession(b) + const hm = new HistoryManager() + await hm.init() + hm.setSessionId('sa') + hm.setCurrentChatId('ca') + await hm.saveChat( + [{ role: 'user', content: 'hello' } as never], + [{ role: 'user', content: 'hello' } as never] + ) + pushMock.mockResolvedValueOnce({ + enabled: true, + storage_id: 'bucket-1', + results: [{ id: 'sa' }, { id: 'sb' }] + }) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(1) + + // Only `sa`'s record changes, and the answer names a new storage: it holds `sb` + // nowhere and `sa` only in the part that went, so both are backed up whole again. + await putSession({ ...a, summary: 'changed' }) + pushMock.mockResolvedValueOnce({ + enabled: true, + storage_id: 'bucket-2', + results: [{ id: 'sa' }] + }) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(2) + expect(pushMock.mock.calls[1][0].requestBody.sessions[0].chats).toBeUndefined() + expect(await pendingDirty()).toEqual(['sa', 'sb']) + + pushMock.mockResolvedValueOnce({ + enabled: true, + storage_id: 'bucket-2', + results: [{ id: 'sa' }, { id: 'sb' }] + }) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(3) + const entries = pushMock.mock.calls[2][0].requestBody.sessions + expect(entries.map((s: { id: string }) => s.id).sort()).toEqual(['sa', 'sb']) + expect(entries.every((s: { head?: unknown }) => s.head !== undefined)).toBe(true) + expect(entries.find((s: { id: string }) => s.id === 'sa').chats).toHaveLength(1) + expect(await pendingDirty()).toEqual([]) + }) + + /** A session with three chats of 1.5 MB (record ~3 MB each): its entry splits past the + * 8 MB request target, so it spans two requests. */ + async function splitSession(id: string): Promise { + const s: Session = { id, name: 'session-1', createdAt: 1, workspace_id: 'ws', chatId: 'c1' } + sessionState.sessions = [s] + await putSession(s) + const hm = new HistoryManager() + await hm.init() + hm.setSessionId(id) + const big = 'x'.repeat(1.5 * 1024 * 1024) + for (const cid of ['c1', 'c2', 'c3']) { + hm.setCurrentChatId(cid) + await hm.saveChat( + [{ role: 'user', content: big } as never], + [{ role: 'user', content: big } as never] + ) + } + } + + it('settles nothing of a session whose parts were answered from different storages', async () => { + await splitSession('sp') + pushMock + .mockResolvedValueOnce({ enabled: true, storage_id: 'bucket-1', results: [{ id: 'sp' }] }) + .mockResolvedValueOnce({ enabled: true, storage_id: 'bucket-2', results: [{ id: 'sp' }] }) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(2) + // The first part opens the session whole with its head; only the last completes + // the entry server-side. + const [first, last] = pushMock.mock.calls.map((c) => c[0].requestBody.sessions[0]) + expect(first.partial).toBe(true) + expect(first.whole).toBe(true) + expect(typeof first.push).toBe('string') + expect(first.opens).toBe(true) + expect(first.head).toBeDefined() + expect(last.partial).toBeUndefined() + expect(last.whole).toBe(true) + expect(last.push).toBe(first.push) + expect(last.opens).toBeUndefined() + expect(last.head).toBeUndefined() + expect(await pendingDirty()).toEqual(['sp']) + expect(await __syncRowsForTesting(EMAIL)).toEqual([]) + }) + + it('names an incremental push split over requests on each of its parts', async () => { + await splitSession('si') + pushMock.mockResolvedValue({ enabled: true, results: [{ id: 'si' }] }) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(2) + expect(await pendingDirty()).toEqual([]) + // Every chat changes: the update spans two requests again, incremental this time. + const hm = new HistoryManager() + await hm.init() + hm.setSessionId('si') + const big = 'y'.repeat(1.5 * 1024 * 1024) + for (const cid of ['c1', 'c2', 'c3']) { + hm.setCurrentChatId(cid) + await hm.saveChat( + [{ role: 'user', content: big } as never], + [{ role: 'user', content: big } as never] + ) + } + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(4) + const [first, last] = pushMock.mock.calls.slice(2).map((c) => c[0].requestBody.sessions[0]) + expect(first.whole).toBeUndefined() + expect(first.partial).toBe(true) + expect(typeof first.push).toBe('string') + expect(first.opens).toBe(true) + expect(last.partial).toBeUndefined() + expect(last.push).toBe(first.push) + expect(last.opens).toBeUndefined() + expect(await pendingDirty()).toEqual([]) + hm.close() + }) + + it('retires a removal only once the storage holding the backup answered it', async () => { + const s: Session = { id: 'sr2', name: 'session-1', createdAt: 1, workspace_id: 'ws' } + sessionState.sessions = [s] + await putSession(s) + pushMock.mockResolvedValueOnce({ enabled: true, storage_id: 'A', results: [{ id: 'sr2' }] }) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(1) + + // The workspace moved to another storage before the delete: the copy in A stays. + deleteSession('sr2') + await flush() + pushMock.mockResolvedValueOnce({ enabled: true, storage_id: 'B', results: [{ id: 'sr2' }] }) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(2) + expect(removalKeys()).toEqual(['r::sr2::ws']) + expect((await __syncRowsForTesting(EMAIL)).some((r) => r.id === 'sr2')).toBe(true) + + // Back on A, the removal lands. + pushMock.mockResolvedValueOnce({ enabled: true, storage_id: 'A', results: [{ id: 'sr2' }] }) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(3) + expect(pushMock.mock.calls[2][0].requestBody.removed).toEqual(['sr2']) + expect(removalKeys()).toEqual([]) + expect((await __syncRowsForTesting(EMAIL)).some((r) => r.id === 'sr2')).toBe(false) + }) + + it('removes a deleted session from every storage that holds a copy of it', async () => { + const s: Session = { id: 'sr3', name: 'session-1', createdAt: 1, workspace_id: 'ws' } + sessionState.sessions = [s] + await putSession(s) + pushMock.mockResolvedValueOnce({ enabled: true, storage_id: 'A', results: [{ id: 'sr3' }] }) + await __flushForTesting() + // The workspace moves to B: the row goes stale, the session goes whole to B and + // settles there, and the row remembers the copy A keeps. + await putSession({ ...s, summary: 'changed' }) + pushMock.mockResolvedValue({ enabled: true, storage_id: 'B', results: [{ id: 'sr3' }] }) + await __flushForTesting() + await __flushForTesting() + expect(await pendingDirty()).toEqual([]) + const row = (await __syncRowsForTesting(EMAIL)).find((r) => r.id === 'sr3') + expect(row?.storageId).toBe('B') + expect(row?.alsoIn).toEqual(['A']) + + // Deleted while on B: B's copy goes, and the mark waits for A to answer. + deleteSession('sr3') + await flush() + await __flushForTesting() + expect(removalKeys()).toEqual(['r::sr3::ws']) + const narrowed = (await __syncRowsForTesting(EMAIL)).find((r) => r.id === 'sr3') + expect(narrowed?.storageId).toBe('A') + expect(narrowed?.alsoIn).toBeUndefined() + + // Back on A, the copy there goes too, and only then is the removal done. + pushMock.mockResolvedValue({ enabled: true, storage_id: 'A', results: [{ id: 'sr3' }] }) + await __flushForTesting() + expect(pushMock.mock.lastCall?.[0].requestBody.removed).toEqual(['sr3']) + expect(removalKeys()).toEqual([]) + expect((await __syncRowsForTesting(EMAIL)).some((r) => r.id === 'sr3')).toBe(false) + }) + + it("removes a moved session's old copy from the storage that held it, whatever its old workspace is on now", async () => { + const s: Session = { id: 'mv2', name: 'session-1', createdAt: 1, workspace_id: 'ws' } + sessionState.sessions = [s] + await putSession(s) + pushMock.mockResolvedValueOnce({ enabled: true, storage_id: 'A', results: [{ id: 'mv2' }] }) + await __flushForTesting() + // The old workspace's backups go off, then the session moves: the new workspace's + // row replaces the old one, so the mark carries where the old copy is. + await putSession({ ...s, summary: 'changed' }) + pushMock.mockResolvedValueOnce({ enabled: false, results: [] }) + await __flushForTesting() + await putSession({ ...s, summary: 'changed', workspace_id: 'ws2' }) + pushMock.mockResolvedValueOnce({ enabled: true, storage_id: 'B', results: [{ id: 'mv2' }] }) + await __flushForTesting() + expect(removalKeys()).toEqual(['r::mv2::ws']) + + // On the next page load the old workspace is on again, but on another storage: the + // removal there deletes nothing, and the mark waits for the storage holding the copy. + __resetMirrorForTesting() + pushMock.mockResolvedValueOnce({ enabled: true, storage_id: 'C', results: [{ id: 'mv2' }] }) + await __flushForTesting() + expect(pushMock.mock.lastCall?.[0].requestBody.removed).toEqual(['mv2']) + expect(removalKeys()).toEqual(['r::mv2::ws']) + pushMock.mockResolvedValueOnce({ enabled: true, storage_id: 'A', results: [{ id: 'mv2' }] }) + await __flushForTesting() + expect(pushMock.mock.lastCall?.[0].requestBody.removed).toEqual(['mv2']) + expect(removalKeys()).toEqual([]) + expect((await __syncRowsForTesting(EMAIL)).find((r) => r.id === 'mv2')?.ws).toBe('ws2') + }) + + it('takes a bumped backup generation as a new storage for the rows, not for a removal', async () => { + const a: Session = { id: 'ga', name: 'session-1', createdAt: 1, workspace_id: 'ws' } + const b: Session = { id: 'gb', name: 'session-2', createdAt: 2, workspace_id: 'ws' } + sessionState.sessions = [a, b] + await putSession(a) + await putSession(b) + pushMock.mockResolvedValueOnce({ + enabled: true, + storage_id: 'A', + backup_generation: 0, + results: [{ id: 'ga' }, { id: 'gb' }] + }) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(1) + + // The key was rotated before the delete: the removal is done (the old generation is + // gone with the rotation), and `gb` goes whole again under the new one. + deleteSession('ga') + await flush() + pushMock.mockResolvedValueOnce({ + enabled: true, + storage_id: 'A', + backup_generation: 1, + results: [{ id: 'ga' }] + }) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(2) + expect(pushMock.mock.calls[1][0].requestBody.removed).toEqual(['ga']) + expect(removalKeys()).toEqual([]) + expect(await pendingDirty()).toEqual(['gb']) + }) + + it('sends a session whole again once the server says its head is gone', async () => { + const s: Session = { + id: 'sh', + name: 'session-1', + createdAt: 1, + workspace_id: 'ws', + chatId: 'c1' + } + sessionState.sessions = [s] + await putSession(s) + const hm = new HistoryManager() + await hm.init() + hm.setSessionId('sh') + hm.setCurrentChatId('c1') + await hm.saveChat( + [{ role: 'user', content: 'a' } as never], + [{ role: 'user', content: 'a' } as never] + ) + pushMock.mockResolvedValueOnce({ enabled: true, results: [{ id: 'sh' }] }) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(1) + + // Another device removed the backup; the next chat-only push finds no head there. + hm.setCurrentChatId('c2') + await hm.saveChat( + [{ role: 'user', content: 'ab' } as never], + [{ role: 'user', content: 'ab' } as never] + ) + pushMock.mockResolvedValueOnce({ enabled: true, results: [{ id: 'sh', needs_whole: true }] }) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(2) + expect(pushMock.mock.calls[1][0].requestBody.sessions[0].whole).toBeUndefined() + expect(await pendingDirty()).toEqual(['sh']) + pushMock.mockResolvedValueOnce({ enabled: true, results: [{ id: 'sh' }] }) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(3) + const whole = pushMock.mock.calls[2][0].requestBody.sessions[0] + expect(whole.whole).toBe(true) + expect(whole.push).toBeUndefined() + expect(whole.head).toBeDefined() + expect(whole.chats.map((c: { id: string }) => c.id).sort()).toEqual(['c1', 'c2']) + expect(await pendingDirty()).toEqual([]) + hm.close() + }) + + it('holds the rest of a session back once the server refused a part of it', async () => { + await splitSession('sf') + pushMock.mockResolvedValueOnce({ enabled: true, results: [{ id: 'sf', error: 'boom' }] }) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(1) + expect(await pendingDirty()).toEqual(['sf']) + expect(await __syncRowsForTesting(EMAIL)).toEqual([]) + }) + + it('keeps an edit whose dirty mark cannot be written while the session has no row yet', async () => { + const s: Session = { id: 'sn', name: 'session-1', createdAt: 1, workspace_id: 'ws' } + sessionState.sessions = [s] + await putSession(s) + // The first push is held open; storage is full for the edit that lands meanwhile. + let release!: (value: unknown) => void + pushMock.mockImplementationOnce(() => new Promise((r) => (release = r))) + const inFlight = __flushForTesting() + await vi.waitFor(() => expect(pushMock).toHaveBeenCalledTimes(1)) + const setItem = localStorage.setItem.bind(localStorage) + localStorage.setItem = (key: string, value: string) => { + if (key.includes('::d::')) throw new Error('QuotaExceededError') + setItem(key, value) + } + try { + await putSession({ ...s, summary: 'second' }) + } finally { + localStorage.setItem = setItem + } + release({ enabled: true, results: [{ id: 'sn' }] }) + await inFlight + // The row the push wrote took the bump over, so the edit is still pending. + expect((await __syncRowsForTesting(EMAIL)).find((r) => r.id === 'sn')?.extraV).toBe(1) + expect(await pendingDirty()).toEqual(['sn']) + pushMock.mockResolvedValueOnce({ enabled: true, results: [{ id: 'sn' }] }) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(2) + expect(pushMock.mock.calls[1][0].requestBody.sessions[0].head.summary).toBe('second') + expect(await pendingDirty()).toEqual([]) + }) + + it('carries an edit on the sync row when its dirty mark cannot be written, even during a push', async () => { + pushMock.mockResolvedValueOnce({ enabled: true, results: [{ id: 'sm' }] }) + const s: Session = { id: 'sm', name: 'session-1', createdAt: 1, workspace_id: 'ws' } + sessionState.sessions = [s] + await putSession(s) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(1) + + // An edit's push is held open; storage is full for the edit that follows. + let release!: (value: unknown) => void + pushMock.mockImplementationOnce(() => new Promise((r) => (release = r))) + await putSession({ ...s, summary: 'first' }) + const inFlight = __flushForTesting() + await vi.waitFor(() => expect(pushMock).toHaveBeenCalledTimes(2)) + const setItem = localStorage.setItem.bind(localStorage) + localStorage.setItem = (key: string, value: string) => { + if (key.includes('::d::')) throw new Error('QuotaExceededError') + setItem(key, value) + } + try { + await putSession({ ...s, summary: 'second' }) + } finally { + localStorage.setItem = setItem + } + await vi.waitFor(async () => + expect((await __syncRowsForTesting(EMAIL)).find((r) => r.id === 'sm')?.extraV).toBe(1) + ) + release({ enabled: true, results: [{ id: 'sm' }] }) + await inFlight + // The push's own row write kept the bump, so the later edit is still pending. + expect(await pendingDirty()).toEqual(['sm']) + pushMock.mockResolvedValueOnce({ enabled: true, results: [{ id: 'sm' }] }) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(3) + expect(pushMock.mock.calls[2][0].requestBody.sessions[0].head.summary).toBe('second') + expect(await pendingDirty()).toEqual([]) + }) + + it('files a mark under the user whose store the write landed in', async () => { + markSessionDirty('sw', undefined, 'other@x.com') + expect(localStorage.getItem('windmill_sessions_mirror_pending::other@x.com::d::sw')).toBe('1') + expect(pendingKeys()).toEqual([]) + + // The mark of another user that cannot be written goes to that user's own rows. + const { openDB } = await import('idb') + const theirs = await openDB('windmill-sessions-mirror::other@x.com', 1, { + upgrade: (db) => db.createObjectStore('sync', { keyPath: 'id' }) + }) + await theirs.put('sync', { id: 'so', ws: 'ws', head: 'h', chats: {}, images: {}, flushedV: 1 }) + theirs.close() + const setItem = localStorage.setItem.bind(localStorage) + localStorage.setItem = (key: string, value: string) => { + if (key.includes('::d::')) throw new Error('QuotaExceededError') + setItem(key, value) + } + try { + markSessionDirty('so', undefined, 'other@x.com') + } finally { + localStorage.setItem = setItem + } + await vi.waitFor(async () => { + const db = await openDB('windmill-sessions-mirror::other@x.com', 1) + try { + expect((await db.get('sync', 'so'))?.extraV).toBe(1) + } finally { + db.close() + } + }) + }) + + it('keeps the marks when the server refuses a request, for the next page load', async () => { + const s: Session = { id: 's4', name: 'session-4', createdAt: 1, workspace_id: 'ws' } + sessionState.sessions = [s] + await putSession(s) + + const { ApiError } = await import('$lib/gen') + pushMock.mockRejectedValue( + new ApiError({ method: 'POST', url: '' } as never, { status: 400 } as never, 'quota') + ) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(1) + // Nothing more for this page, marks untouched by the follow-up flush. + await putSession({ ...s, summary: 'changed' }) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(1) + expect(await pendingDirty()).toEqual(['s4']) + }) +}) + +describe('sessionMirror restore', () => { + const backup = { + id: 's9', + head: { id: 's9', workspace_id: 'ws', createdAt: 5, chatId: 'c9', summary: 'remote' }, + chats: [ + { + id: 'c9', + record: { + id: 'c9', + sessionId: 's9', + title: 't', + lastModified: 7, + actualMessages: [], + displayMessages: [{ role: 'user', content: 'hi' }] + } + } + ], + images: [] + } + + it('brings back a session the browser lacks, and records nothing for one whose chats could not be written', async () => { + listMock.mockResolvedValue({ + enabled: true, + sessions: [{ id: 's9', updated_at: '2026-09-14T00:00:00Z' }] + }) + pullMock.mockResolvedValue({ enabled: true, sessions: [backup], deferred: [] }) + usersWorkspaceStore.set({ email: EMAIL, workspaces: [] } as never) + + // A removal pending for another workspace (the session moved here from it) does + // not stand in the way of restoring this workspace's copy. + localStorage.setItem(`${PENDING_PREFIX}r::s9::elsewhere`, '1') + chatImport.unavailable = true + restoreSessionBackups('ws') + await __settleForTesting() + expect(pullMock).toHaveBeenCalledTimes(1) + expect(sessionState.sessions.map((s) => s.id)).toEqual([]) + // Not recorded as restored: the next restore tries again, and no flush can push a + // transcript-less copy over the backup. + __resetMirrorForTesting() + chatImport.unavailable = false + restoreSessionBackups('ws') + await __settleForTesting() + expect(pullMock).toHaveBeenCalledTimes(2) + await vi.waitFor(() => expect(sessionState.sessions.map((s) => s.id)).toEqual(['s9'])) + const restored = sessionState.sessions[0] + expect(restored.name).toBe('session-1') + expect(restored.summary).toBe('remote') + expect(restored.lastSeenCount).toBe(1) + expect((await readStoredChat('c9', EMAIL))?.displayMessages).toHaveLength(1) + + // Restored state is what the backup holds: nothing to push (the other workspace's + // removal is its own request, not part of this check). + localStorage.removeItem(`${PENDING_PREFIX}r::s9::elsewhere`) + await __flushForTesting() + expect(pushMock).not.toHaveBeenCalled() + }) + + it('imports a session that came in pages only once the last page arrived', async () => { + listMock.mockResolvedValue({ + enabled: true, + sessions: [{ id: 's9', updated_at: '2026-09-14T00:00:00Z' }] + }) + const cursor = { id: 's9', images: false, after: 'sessions/s9/chats/c9.json' } + const c9b = { ...backup.chats[0], id: 'c9b', record: { ...backup.chats[0].record, id: 'c9b' } } + pullMock + .mockResolvedValueOnce({ + enabled: true, + sessions: [{ ...backup, next: cursor }], + deferred: [] + }) + .mockResolvedValueOnce({ + enabled: true, + sessions: [{ ...backup, chats: [c9b] }], + deferred: [] + }) + usersWorkspaceStore.set({ email: EMAIL, workspaces: [] } as never) + restoreSessionBackups('ws') + await __settleForTesting() + expect(pullMock).toHaveBeenCalledTimes(2) + expect(pullMock.mock.calls[1][0].requestBody).toEqual({ ids: ['s9'], resume: cursor }) + await vi.waitFor(() => expect(sessionState.sessions.map((s) => s.id)).toEqual(['s9'])) + expect((await readStoredChat('c9', EMAIL))?.displayMessages).toHaveLength(1) + expect((await readStoredChat('c9b', EMAIL))?.id).toBe('c9b') + }) + + it('starts a session over when its only page was read while the backup moved', async () => { + listMock.mockResolvedValue({ + enabled: true, + sessions: [{ id: 's9', updated_at: '2026-09-14T00:00:00Z' }] + }) + const c9b = { ...backup.chats[0], id: 'c9b', record: { ...backup.chats[0].record, id: 'c9b' } } + pullMock + .mockResolvedValueOnce({ + enabled: true, + sessions: [{ ...backup, chats: [backup.chats[0]], moved: true }], + deferred: [] + }) + .mockResolvedValueOnce({ + enabled: true, + sessions: [{ ...backup, chats: [backup.chats[0], c9b] }], + deferred: [] + }) + usersWorkspaceStore.set({ email: EMAIL, workspaces: [] } as never) + restoreSessionBackups('ws') + await __settleForTesting() + expect(pullMock).toHaveBeenCalledTimes(2) + await vi.waitFor(() => expect(sessionState.sessions.map((s) => s.id)).toEqual(['s9'])) + expect((await readStoredChat('c9b', EMAIL))?.id).toBe('c9b') + }) + + it('brings a session two workspaces of the family list back from the copy that moved last', async () => { + // Both copies carry the same modification time (a store reports them coarsely): the + // move count tells them apart. + listMock.mockImplementation(async ({ workspace }: { workspace: string }) => ({ + enabled: true, + sessions: [ + { id: 's9', updated_at: '2026-09-14T00:00:00Z', epoch: workspace === 'ws' ? 0 : 1 } + ] + })) + pullMock.mockImplementation(async ({ workspace }: { workspace: string }) => ({ + enabled: true, + sessions: [{ ...backup, head: { ...backup.head, workspace_id: workspace, moves: 1 } }], + deferred: [] + })) + usersWorkspaceStore.set({ + email: EMAIL, + workspaces: [ + { id: 'ws', name: 'ws', username: 'u' }, + { id: 'ws2', name: 'ws2', username: 'u', parent_workspace_id: 'ws' } + ] + } as never) + restoreSessionBackups('ws') + await __settleForTesting() + // The stale copy in the old workspace is left alone, and does not take the id first. + expect(pullMock.mock.calls.map((c) => c[0].workspace)).toEqual(['ws2']) + await vi.waitFor(() => expect(sessionState.sessions.map((s) => s.id)).toEqual(['s9'])) + expect(sessionState.sessions[0].workspace_id).toBe('ws2') + }) + + it('leaves a session whose later copy showed up elsewhere after the listings, and tries again', async () => { + // The move lands in the other workspace between the family's listings and the pull: + // the listing taken again before the record lands shows it, and the copy about to be + // imported is the stale one. + let ws2Listings = 0 + listMock.mockImplementation(async ({ workspace }: { workspace: string }) => { + if (workspace === 'ws') { + return { + enabled: true, + sessions: [{ id: 's9', updated_at: '2026-09-14T00:00:00Z', epoch: 0 }] + } + } + ws2Listings += 1 + return { + enabled: true, + sessions: + ws2Listings === 1 ? [] : [{ id: 's9', updated_at: '2026-09-14T00:00:00Z', epoch: 1 }] + } + }) + pullMock.mockImplementation(async ({ workspace }: { workspace: string }) => ({ + enabled: true, + sessions: [{ ...backup, head: { ...backup.head, workspace_id: workspace, moves: 1 } }], + deferred: [] + })) + usersWorkspaceStore.set({ + email: EMAIL, + workspaces: [ + { id: 'ws', name: 'ws', username: 'u' }, + { id: 'ws2', name: 'ws2', username: 'u', parent_workspace_id: 'ws' } + ] + } as never) + restoreSessionBackups('ws') + await __settleForTesting() + expect(pullMock.mock.calls.map((c) => c[0].workspace)).toEqual(['ws']) + expect(sessionState.sessions).toEqual([]) + restoreSessionBackups('ws') + await __settleForTesting() + await vi.waitFor(() => expect(sessionState.sessions.map((s) => s.id)).toEqual(['s9'])) + expect(sessionState.sessions[0].workspace_id).toBe('ws2') + }) + + it('checks a family member whose backups were off when the restore started', async () => { + // The other workspace comes on (and gets the moved session) between the family's + // listings and the pull: the listing taken again before the records land covers it. + let ws2Listings = 0 + listMock.mockImplementation(async ({ workspace }: { workspace: string }) => { + if (workspace === 'ws') { + return { + enabled: true, + sessions: [{ id: 's9', updated_at: '2026-09-14T00:00:00Z', epoch: 0 }] + } + } + ws2Listings += 1 + return ws2Listings === 1 + ? { enabled: false, sessions: [] } + : { + enabled: true, + sessions: [{ id: 's9', updated_at: '2026-09-14T00:00:00Z', epoch: 1 }] + } + }) + pullMock.mockImplementation(async ({ workspace }: { workspace: string }) => ({ + enabled: true, + sessions: [{ ...backup, head: { ...backup.head, workspace_id: workspace, moves: 1 } }], + deferred: [] + })) + usersWorkspaceStore.set({ + email: EMAIL, + workspaces: [ + { id: 'ws', name: 'ws', username: 'u' }, + { id: 'ws2', name: 'ws2', username: 'u', parent_workspace_id: 'ws' } + ] + } as never) + restoreSessionBackups('ws') + await __settleForTesting() + expect(sessionState.sessions).toEqual([]) + restoreSessionBackups('ws') + await __settleForTesting() + await vi.waitFor(() => expect(sessionState.sessions.map((s) => s.id)).toEqual(['s9'])) + expect(sessionState.sessions[0].workspace_id).toBe('ws2') + }) + + it('marks nothing at load for a session a restore brought back', async () => { + listMock.mockResolvedValue({ + enabled: true, + sessions: [{ id: 's9', updated_at: '2026-09-14T00:00:00Z', epoch: 0 }] + }) + pullMock.mockResolvedValue({ enabled: true, sessions: [backup], deferred: [] }) + usersWorkspaceStore.set({ email: EMAIL, workspaces: [] } as never) + restoreSessionBackups('ws') + await __settleForTesting() + await vi.waitFor(() => expect(sessionState.sessions.map((s) => s.id)).toEqual(['s9'])) + // The next load's backfill leaves the clean row alone: no mark, no read, no push. + __resetMirrorForTesting() + await __flushForTesting() + expect(pushMock).not.toHaveBeenCalled() + expect(pendingKeys()).toEqual([]) + }) + + it('lists a family of one once, whatever it restores', async () => { + listMock.mockResolvedValue({ + enabled: true, + sessions: [ + { id: 's9', updated_at: '2026-09-14T00:00:00Z', epoch: 0 }, + { id: 's8', updated_at: '2026-09-13T00:00:00Z', epoch: 0 } + ] + }) + pullMock.mockImplementation(async ({ requestBody }: { requestBody: { ids: string[] } }) => ({ + enabled: true, + sessions: requestBody.ids.map((id) => ({ + ...backup, + id, + head: { ...backup.head, id }, + chats: backup.chats.map((c) => ({ + ...c, + id: `${c.id}-${id}`, + record: { ...c.record, id: `${c.id}-${id}`, sessionId: id } + })) + })), + deferred: [] + })) + usersWorkspaceStore.set({ email: EMAIL, workspaces: [] } as never) + restoreSessionBackups('ws') + await __settleForTesting() + await vi.waitFor(() => + expect(sessionState.sessions.map((s) => s.id).sort()).toEqual(['s8', 's9']) + ) + expect(listMock).toHaveBeenCalledTimes(1) + }) + + it('restores nothing of a family one of whose workspaces could not be listed, and tries again', async () => { + listMock + .mockImplementationOnce(async () => ({ + enabled: true, + sessions: [{ id: 's9', updated_at: '2026-09-14T00:00:00Z', epoch: 0 }] + })) + .mockRejectedValueOnce(new Error('offline')) + .mockImplementation(async ({ workspace }: { workspace: string }) => ({ + enabled: true, + sessions: [ + { id: 's9', updated_at: '2026-09-14T00:00:00Z', epoch: workspace === 'ws' ? 0 : 1 } + ] + })) + pullMock.mockImplementation(async ({ workspace }: { workspace: string }) => ({ + enabled: true, + sessions: [{ ...backup, head: { ...backup.head, workspace_id: workspace, moves: 1 } }], + deferred: [] + })) + usersWorkspaceStore.set({ + email: EMAIL, + workspaces: [ + { id: 'ws', name: 'ws', username: 'u' }, + { id: 'ws2', name: 'ws2', username: 'u', parent_workspace_id: 'ws' } + ] + } as never) + restoreSessionBackups('ws') + await __settleForTesting() + // The copy that listed could be the stale one: nothing is imported this time. + expect(pullMock).not.toHaveBeenCalled() + expect(sessionState.sessions).toEqual([]) + restoreSessionBackups('ws') + await __settleForTesting() + expect(pullMock.mock.calls.map((c) => c[0].workspace)).toEqual(['ws2']) + await vi.waitFor(() => expect(sessionState.sessions.map((s) => s.id)).toEqual(['s9'])) + }) + + it('starts a session over when its backup moved between two pages', async () => { + listMock.mockResolvedValue({ + enabled: true, + sessions: [{ id: 's9', updated_at: '2026-09-14T00:00:00Z' }] + }) + const cursor = { id: 's9', images: false, after: 'sessions/s9/chats/c9.json' } + const c9b = { ...backup.chats[0], id: 'c9b', record: { ...backup.chats[0].record, id: 'c9b' } } + const c9a = { ...backup.chats[0], id: 'c9a', record: { ...backup.chats[0].record, id: 'c9a' } } + pullMock + // The first attempt: a chat sorting before the cursor lands between the pages. + .mockResolvedValueOnce({ + enabled: true, + sessions: [{ ...backup, next: cursor, listing: 'L1' }], + deferred: [] + }) + .mockResolvedValueOnce({ + enabled: true, + sessions: [{ ...backup, chats: [c9b], listing: 'L2' }], + deferred: [] + }) + // The second attempt sees the whole of it. + .mockResolvedValueOnce({ + enabled: true, + sessions: [{ ...backup, chats: [c9a, backup.chats[0]], next: cursor, listing: 'L2' }], + deferred: [] + }) + .mockResolvedValueOnce({ + enabled: true, + sessions: [{ ...backup, chats: [c9b], listing: 'L2' }], + deferred: [] + }) + usersWorkspaceStore.set({ email: EMAIL, workspaces: [] } as never) + restoreSessionBackups('ws') + await __settleForTesting() + expect(pullMock).toHaveBeenCalledTimes(4) + expect(pullMock.mock.calls[2][0].requestBody).toEqual({ ids: ['s9'], resume: undefined }) + await vi.waitFor(() => expect(sessionState.sessions.map((s) => s.id)).toEqual(['s9'])) + expect((await readStoredChat('c9a', EMAIL))?.id).toBe('c9a') + expect((await readStoredChat('c9b', EMAIL))?.id).toBe('c9b') + }) + + it('restores nothing without Web Locks, and still backs up', async () => { + setWebLocks(undefined) + listMock.mockResolvedValue({ + enabled: true, + sessions: [{ id: 's9', updated_at: '2026-09-14T00:00:00Z' }] + }) + usersWorkspaceStore.set({ email: EMAIL, workspaces: [] } as never) + restoreSessionBackups('ws') + await __settleForTesting() + expect(listMock).not.toHaveBeenCalled() + expect(sessionState.sessions).toEqual([]) + const s: Session = { id: 'sl', name: 'session-1', createdAt: 1, workspace_id: 'ws' } + sessionState.sessions = [s] + await putSession(s) + pushMock.mockResolvedValueOnce({ enabled: true, results: [{ id: 'sl' }] }) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(1) + }) + + it('leaves a session for the next restore when what an earlier one staged cannot be pruned', async () => { + listMock.mockResolvedValue({ + enabled: true, + sessions: [{ id: 's9', updated_at: '2026-09-14T00:00:00Z' }] + }) + // An earlier restore was cut short after staging a chat the backup has since dropped. + const { importStoredChats } = await import('../copilot/chat/HistoryManager.svelte') + await importStoredChats([{ ...backup.chats[0].record, id: 'cx' } as never], [], EMAIL, true) + const staging = { + id: 's9', + ws: 'ws', + head: '', + chats: {}, + images: {}, + staging: { chats: ['cx'], images: [], items: [], versions: [] } + } + await __writeSyncForTesting([staging], EMAIL) + pullMock.mockResolvedValue({ enabled: true, sessions: [backup], deferred: [] }) + chatImport.pruneFails = true + usersWorkspaceStore.set({ email: EMAIL, workspaces: [] } as never) + restoreSessionBackups('ws') + await __settleForTesting() + // No record, so no flush can push the stale chat back, and the staging row stays, + // now naming what this page wrote too. + expect(sessionState.sessions).toEqual([]) + expect((await readStoredChat('cx', EMAIL))?.id).toBe('cx') + expect( + (await __syncRowsForTesting(EMAIL)).find((r) => r.id === 's9')?.staging?.chats?.sort() + ).toEqual(['c9', 'cx']) + + // The restore after prunes and brings the session back. + chatImport.pruneFails = false + __resetMirrorForTesting() + restoreSessionBackups('ws') + await __settleForTesting() + await vi.waitFor(() => expect(sessionState.sessions.map((s) => s.id)).toEqual(['s9'])) + expect(await readStoredChat('cx', EMAIL)).toBeUndefined() + expect((await __syncRowsForTesting(EMAIL)).find((r) => r.id === 's9')?.staging).toBeUndefined() + }) + + it('never writes an older record over a newer one, and prunes only what it is told', async () => { + const { importStoredChats, pruneSessionChats } = await import( + '../copilot/chat/HistoryManager.svelte' + ) + const chat = (lastModified: number, title: string) => + ({ ...backup.chats[0].record, lastModified, title }) as never + await importStoredChats([chat(20, 'newer')], [], EMAIL, true) + await importStoredChats([chat(10, 'older')], [], EMAIL, true) + expect((await readStoredChat('c9', EMAIL))?.title).toBe('newer') + await importStoredChats( + [{ ...backup.chats[0].record, id: 'c9b', lastModified: 30 } as never], + [], + EMAIL, + true + ) + // A prune names what goes; nothing else of the session is touched. + await pruneSessionChats('s9', new Set(['c9']), new Set(), EMAIL) + expect(await readStoredChat('c9', EMAIL)).toBeUndefined() + expect((await readStoredChat('c9b', EMAIL))?.id).toBe('c9b') + }) + + it('keeps an image whose chat came on an earlier page, and restages after a page failed', async () => { + listMock.mockResolvedValue({ + enabled: true, + sessions: [{ id: 's9', updated_at: '2026-09-14T00:00:00Z' }] + }) + const cursor = { id: 's9', images: true, after: '' } + const imagePage = { + ...backup, + chats: [], + images: [{ chat_id: 'c9', id: 'i9', data_url: IMAGE }] + } + // The first restore is cut short after staging the chats and an artifact with two + // versions. + const gone = { ...backup.chats[0], id: 'cx', record: { ...backup.chats[0].record, id: 'cx' } } + const item = { id: 'a1', sessionId: 's9', kind: 'markdown', name: 'a', content: 'x' } + const version = (n: number) => ({ + key: `a1:${n}`, + artifactId: 'a1', + version: n, + name: 'a', + content: 'x', + savedAt: n + }) + pullMock + .mockResolvedValueOnce({ + enabled: true, + sessions: [ + { + ...backup, + chats: [...backup.chats, gone], + artifacts: { items: [item], versions: [version(1), version(2)] }, + next: cursor + } + ], + deferred: [] + }) + .mockRejectedValueOnce(new Error('offline')) + usersWorkspaceStore.set({ email: EMAIL, workspaces: [] } as never) + restoreSessionBackups('ws') + await __settleForTesting() + expect(pullMock).toHaveBeenCalledTimes(2) + expect(sessionState.sessions.map((s) => s.id)).toEqual([]) + expect((await readStoredChat('c9', EMAIL))?.title).toBe('t') + expect((await readStoredChat('cx', EMAIL))?.id).toBe('cx') + + // The backup moved on meanwhile: the retry takes the newer chat over the staged one + // and drops the chat the backup no longer has. + __resetMirrorForTesting() + const newer = { + ...backup, + chats: [{ ...backup.chats[0], record: { ...backup.chats[0].record, title: 'newer' } }], + artifacts: { items: [item], versions: [version(1)] } + } + pullMock + .mockResolvedValueOnce({ + enabled: true, + sessions: [{ ...newer, next: cursor }], + deferred: [] + }) + .mockResolvedValueOnce({ enabled: true, sessions: [imagePage], deferred: [] }) + restoreSessionBackups('ws') + await __settleForTesting() + await vi.waitFor(() => expect(sessionState.sessions.map((s) => s.id)).toEqual(['s9'])) + expect((await readStoredChat('c9', EMAIL))?.title).toBe('newer') + expect(await readStoredChat('cx', EMAIL)).toBeUndefined() + const { readImageDataUrl } = await import('../copilot/chat/HistoryManager.svelte') + expect(await readImageDataUrl('i9', EMAIL)).toBe(IMAGE) + // The version the backup no longer has went with the chat it no longer has. + const { readSessionArtifacts } = await import('../copilot/chat/artifacts/artifactsDB') + const artifacts = await readSessionArtifacts('s9', EMAIL) + expect(artifacts?.items.map((i) => i.id)).toEqual(['a1']) + expect(artifacts?.versions.map((v) => v.key)).toEqual(['a1:1']) + }) +}) diff --git a/frontend/src/lib/components/sessions/sessionMirrorPlan.test.ts b/frontend/src/lib/components/sessions/sessionMirrorPlan.test.ts new file mode 100644 index 0000000000..aa3c8bf735 --- /dev/null +++ b/frontend/src/lib/components/sessions/sessionMirrorPlan.test.ts @@ -0,0 +1,204 @@ +import { describe, expect, it } from 'vitest' +import { + artifactsFingerprint, + headSig, + jsonBytes, + planSessionPush, + splitEntry, + type ChatSnapshot, + type MirrorSyncState +} from './sessionMirrorPlan' +import type { Session } from './sessionState.svelte' + +function session(over: Partial = {}): Session { + return { id: 's1', name: 'session-1', createdAt: 1, workspace_id: 'ws', chatId: 'c1', ...over } +} + +function chat(id: string, lastModified: number, imageIds: string[] = []): ChatSnapshot { + return { id, lastModified, record: { id, lastModified }, imageIds } +} + +const noArtifacts = { items: [], versions: [] } + +function synced(over: Partial = {}): MirrorSyncState { + return { + id: 's1', + ws: 'ws', + head: headSig(session()), + chats: { c1: 10 }, + images: { i1: 'c1' }, + artifacts: artifactsFingerprint(noArtifacts), + ...over + } +} + +describe('planSessionPush', () => { + it('pushes everything for a session never backed up, and nothing for an unsent draft', () => { + const plan = planSessionPush({ + session: session(), + chats: [chat('c1', 10, ['i1'])], + artifacts: noArtifacts + }) + expect(plan?.entry?.head?.id).toBe('s1') + expect(plan?.entry?.chats?.map((c) => c.id)).toEqual(['c1']) + expect(plan?.images).toEqual([{ chat_id: 'c1', id: 'i1' }]) + // Nothing to store yet, so no artifacts object either. + expect(plan?.entry?.artifacts).toBeUndefined() + expect(plan?.next).toEqual(synced()) + + expect( + planSessionPush({ + session: session({ workspace_id: undefined, pending_workspace_id: 'ws' }), + chats: [], + artifacts: noArtifacts + }) + ).toBeUndefined() + }) + + it('sends nothing when only the fields reading a session bumps changed', () => { + const plan = planSessionPush({ + session: session({ lastSeenCount: 7, lastActivityAt: 99, name: 'session-9' }), + chats: [chat('c1', 10, ['i1'])], + artifacts: noArtifacts, + sync: synced() + }) + expect(plan?.entry).toBeUndefined() + expect(plan?.images).toEqual([]) + }) + + it('carries only the chat whose lastModified moved, plus its new images', () => { + const plan = planSessionPush({ + session: session(), + chats: [chat('c1', 10, ['i1']), chat('c2', 20, ['i2'])], + artifacts: noArtifacts, + sync: synced() + }) + expect(plan?.entry?.head).toBeUndefined() + expect(plan?.entry?.chats?.map((c) => c.id)).toEqual(['c2']) + expect(plan?.images).toEqual([{ chat_id: 'c2', id: 'i2' }]) + expect(plan?.next.chats).toEqual({ c1: 10, c2: 20 }) + expect(plan?.next.images).toEqual({ i1: 'c1', i2: 'c2' }) + }) + + it('deletes the copy of a chat that grew too large to back up, instead of keeping a stale one', () => { + const plan = planSessionPush({ + session: session(), + chats: [{ id: 'c1', lastModified: 11, imageIds: ['i1'], omitted: true }], + artifacts: noArtifacts, + sync: synced() + }) + expect(plan?.entry?.delete_chats).toEqual(['c1']) + expect(plan?.entry?.chats).toBeUndefined() + expect(plan?.images).toEqual([]) + expect(plan?.next.chats).toEqual({}) + expect(plan?.next.images).toEqual({}) + }) + + it('carries deletes past the per-entry cap over to the next push', () => { + const prevChats = Object.fromEntries(Array.from({ length: 1005 }, (_, i) => [`c${i}`, 10])) + const plan = planSessionPush({ + session: session(), + chats: [], + artifacts: noArtifacts, + sync: synced({ chats: prevChats, images: {} }) + }) + expect(plan?.entry?.delete_chats).toHaveLength(1000) + // Still listed as pushed, so the next plan finds them gone again, and the session + // stays marked for that plan. + expect(Object.keys(plan?.next.chats ?? {})).toHaveLength(5) + expect(plan?.carried).toBe(true) + }) + + it('deletes a chat that is gone and an image its chat evicted', () => { + const plan = planSessionPush({ + session: session(), + chats: [chat('c1', 11, [])], + artifacts: noArtifacts, + sync: synced({ chats: { c1: 10, c2: 20 }, images: { i1: 'c1', i2: 'c2' } }) + }) + expect(plan?.entry?.delete_chats).toEqual(['c2']) + // i2 goes with c2 server-side; only c1's evicted image is deleted on its own. + expect(plan?.entry?.delete_images).toEqual([{ chat_id: 'c1', id: 'i1' }]) + }) + + it('moves a session as a full push to the new workspace and a removal from the old', () => { + const plan = planSessionPush({ + session: session({ workspace_id: 'ws2' }), + chats: [chat('c1', 10, ['i1'])], + artifacts: noArtifacts, + sync: synced() + }) + expect(plan?.workspaceId).toBe('ws2') + expect(plan?.removeFrom).toBe('ws') + expect(plan?.entry?.head?.workspace_id).toBe('ws2') + expect(plan?.entry?.chats?.map((c) => c.id)).toEqual(['c1']) + expect(plan?.images).toEqual([{ chat_id: 'c1', id: 'i1' }]) + expect(plan?.next.ws).toBe('ws2') + }) + + it('pushes artifacts when their fingerprint changes, including emptying them', () => { + const items = [ + { + id: 'a1', + sessionId: 's1', + kind: 'md' as const, + name: 'notes', + content: 'x', + createdAt: 1, + updatedAt: 2, + version: 1 + } + ] + const withArtifact = planSessionPush({ + session: session(), + chats: [chat('c1', 10, ['i1'])], + artifacts: { items, versions: [] }, + sync: synced() + }) + expect(withArtifact?.entry?.artifacts).toEqual({ items, versions: [] }) + + const emptied = planSessionPush({ + session: session(), + chats: [chat('c1', 10, ['i1'])], + artifacts: noArtifacts, + sync: synced({ artifacts: artifactsFingerprint({ items, versions: [] }) }) + }) + expect(emptied?.entry?.artifacts).toEqual(noArtifacts) + }) +}) + +describe('jsonBytes', () => { + it('counts the bytes the request carries, not UTF-16 code units', () => { + expect(jsonBytes('ab')).toBe(4) + expect(jsonBytes('日本')).toBe(8) + expect(jsonBytes('😀')).toBe(6) + }) +}) + +describe('splitEntry', () => { + it('splits an oversized entry into chat-only parts, the head riding on the last', () => { + const big = (id: string) => ({ id, record: { id, text: 'x'.repeat(150) } }) + const entry = { + id: 's', + head: { id: 's' }, + chats: [big('c1'), big('c2'), big('c3')], + delete_chats: ['old'] + } + const parts = splitEntry(entry, 200) + expect(parts.map((p) => p.chats?.map((c) => c.id))).toEqual([['c1'], ['c2'], ['c3']]) + expect( + parts.slice(0, -1).every((p) => p.head === undefined && p.delete_chats === undefined) + ).toBe(true) + expect(parts.at(-1)?.head).toEqual({ id: 's' }) + expect(parts.at(-1)?.delete_chats).toEqual(['old']) + // Within the target, or a single chat: nothing to split. + expect(splitEntry(entry, 10_000)).toEqual([entry]) + expect(splitEntry({ id: 's', chats: [big('c1')] }, 10)).toHaveLength(1) + // The server's per-entry chat cap splits too, however small the chats. + const many = { + id: 's', + chats: Array.from({ length: 250 }, (_, i) => ({ id: `c${i}`, record: {} })) + } + expect(splitEntry(many, 1_000_000).map((p) => p.chats?.length)).toEqual([100, 100, 50]) + }) +}) diff --git a/frontend/src/lib/components/sessions/sessionMirrorPlan.ts b/frontend/src/lib/components/sessions/sessionMirrorPlan.ts new file mode 100644 index 0000000000..c61bcf2763 --- /dev/null +++ b/frontend/src/lib/components/sessions/sessionMirrorPlan.ts @@ -0,0 +1,309 @@ +// The pure half of the session backup: given what the local stores hold for one session +// and what was last pushed, decide what the next push carries. Every piece is compared +// against its own marker (a signature for the record, `lastModified` for a chat, the id +// for a write-once image) so a session that only changed locally in ways the backup +// does not keep sends nothing. +import type { AISessionBackupPush } from '$lib/gen' +import { orderedJsonStringify } from '$lib/utils' +import type { Session } from './sessionState.svelte' +import type { ArtifactVersion, PersistedArtifact } from '../copilot/chat/artifacts/artifactsDB' + +/** What the backup remembers of a session after a successful push. */ +export interface MirrorSyncState { + id: string + /** The workspace whose storage holds the backup. */ + ws: string + /** `headSig` of the record pushed. */ + head: string + /** `lastModified` of each chat pushed, by chat id. */ + chats: Record + /** The chat each pushed image belongs to, by image id. */ + images: Record + artifacts?: string + /** The workspace's storage went away after this push: what it holds is unknown, so the + * next push carries everything again. Kept rather than deleted, so a removal still + * knows a backup existed. */ + stale?: boolean + /** The dirty mark's counter this push covered. A mark is pending while its counter is + * above this; retiring it here rather than deleting the mark means a tab bumping the + * counter while another flushes can never have its bump erased. */ + flushedV?: number + /** The user deleted the session and its removal mark could not be written to + * localStorage (full): the row itself carries the removal, until it lands. */ + removed?: boolean + /** The storage the push landed in, as the server names it, and the backup generation + * (bumped by a workspace key rotation) it landed under. A row recorded against another + * storage or generation describes objects the server no longer looks at. */ + storageId?: string + generation?: number + /** Other storages this workspace was on that still hold a copy of the backup (a switch + * leaves the old copy where it was): a removal is done only once each has answered it, + * or a switch back would bring a deleted session back. */ + alsoIn?: string[] + /** Bumps of the dirty mark that localStorage refused, recorded here instead: the mark's + * counter plus this is what a push retires, and every row write keeps it. */ + extraV?: number + /** A restore in progress (or cut short): the pieces it wrote for a session that has no + * record yet, so a later restore deletes the ones the backup no longer has. */ + staging?: { chats: string[]; images: string[]; items: string[]; versions: string[] } +} + +/** + * The part of a session record the backup keeps. Left out on purpose: `name` (a + * per-browser counter), the unsent-draft fields (`pending_*`, `draftPrompt`, + * `autoSendDraftAt`), `workspace_root_id` (derived on import), `transient`, and the two + * fields reading a session bumps (`lastSeenCount`, `lastActivityAt`) — so opening a + * session and reading its new messages never costs a push. + */ +export type SessionHead = Pick< + Session, + | 'id' + | 'workspace_id' + | 'chatId' + | 'summary' + | 'summarySource' + | 'createdAt' + | 'archived' + | 'archivedByWorkspace' + | 'moves' + | 'previewTabs' + | 'activePreviewTabId' + | 'previewCollapsed' + | 'previewSize' +> + +export function sessionHead(s: Session): SessionHead { + const head: SessionHead = { id: s.id, createdAt: s.createdAt } + if (s.workspace_id !== undefined) head.workspace_id = s.workspace_id + if (s.chatId !== undefined) head.chatId = s.chatId + if (s.summary !== undefined) head.summary = s.summary + if (s.summarySource !== undefined) head.summarySource = s.summarySource + if (s.archived !== undefined) head.archived = s.archived + if (s.archivedByWorkspace !== undefined) head.archivedByWorkspace = s.archivedByWorkspace + if (s.moves !== undefined) head.moves = s.moves + if (s.previewTabs !== undefined) head.previewTabs = s.previewTabs + if (s.activePreviewTabId !== undefined) head.activePreviewTabId = s.activePreviewTabId + if (s.previewCollapsed !== undefined) head.previewCollapsed = s.previewCollapsed + if (s.previewSize !== undefined) head.previewSize = s.previewSize + return head +} + +export function headSig(s: Session): string { + return orderedJsonStringify(sessionHead(s)) +} + +export interface ArtifactsSnapshot { + items: PersistedArtifact[] + versions: ArtifactVersion[] +} + +/** Cheap to compute from the rows alone: every edit bumps `updatedAt`, every snapshot has + * its own key, and approving a plan changes `approvedVersion`. */ +export function artifactsFingerprint(a: ArtifactsSnapshot): string { + const items = a.items + .map((i) => `${i.id}:${i.updatedAt}:${i.version ?? 1}:${i.approvedVersion ?? ''}`) + .sort() + const versions = a.versions.map((v) => v.key).sort() + return JSON.stringify([items, versions]) +} + +export interface ChatSnapshot { + id: string + lastModified: number + /** The stored record; absent for a chat that did not change since the last push, whose + * bytes the caller did not read. */ + record?: unknown + imageIds: string[] + /** Too large to back up: planned as if it did not exist, so a copy pushed while it was + * smaller is deleted rather than restored one day as the current transcript. */ + omitted?: boolean +} + +export interface PlanInput { + session: Session + /** Every chat the session owns right now. */ + chats: ChatSnapshot[] + artifacts: ArtifactsSnapshot + sync?: MirrorSyncState +} + +export interface PlannedPush { + workspaceId: string + /** Absent when nothing changed that the backup keeps. */ + entry?: AISessionBackupPush + /** Images the entry needs uploaded, whose bytes the caller loads. */ + images: { chat_id: string; id: string }[] + /** The workspace the session was backed up in before it moved. */ + removeFrom?: string + next: MirrorSyncState + /** Deletes past the per-entry cap were left in `next` for the following push, so the + * session must stay marked once this one lands. */ + carried: boolean + /** Nothing of the session is taken to be in the storage: every piece goes, and the + * first part opens the push whole (see `whole` on the entry). */ + whole: boolean +} + +/** `undefined` for a session with nowhere to go: an unsent draft has no workspace yet. */ +export function planSessionPush(input: PlanInput): PlannedPush | undefined { + const { session, chats, artifacts } = input + const workspaceId = session.workspace_id + if (!workspaceId) return undefined + // A move is a full push into the new workspace's storage; the copy in the old one goes. + const prev = input.sync?.ws === workspaceId ? input.sync : undefined + const removeFrom = input.sync && input.sync.ws !== workspaceId ? input.sync.ws : undefined + + const entry: AISessionBackupPush = { id: session.id } + let changed = false + let carried = false + const sig = headSig(session) + if (prev?.head !== sig) { + entry.head = sessionHead(session) + changed = true + } + + const next: MirrorSyncState = { + id: session.id, + ws: workspaceId, + head: sig, + chats: {}, + images: {} + } + const images: { chat_id: string; id: string }[] = [] + const pushedChats: { id: string; record: Record }[] = [] + for (const chat of chats) { + if (chat.omitted) continue + next.chats[chat.id] = chat.lastModified + if (prev?.chats[chat.id] !== chat.lastModified && chat.record !== undefined) { + pushedChats.push({ id: chat.id, record: chat.record as Record }) + } + for (const id of chat.imageIds) { + next.images[id] = chat.id + if (prev?.images[id] === undefined) images.push({ chat_id: chat.id, id }) + } + } + if (pushedChats.length > 0) { + entry.chats = pushedChats + changed = true + } + if (prev) { + const gone = Object.keys(prev.chats).filter((id) => next.chats[id] === undefined) + // An image evicted by the per-chat cap, from a chat that is still there (a deleted + // chat takes its images with it server-side). + const evicted = Object.entries(prev.images).filter( + ([id, chatId]) => next.images[id] === undefined && next.chats[chatId] !== undefined + ) + // Past the server's cap per entry, the rest stays in `next` as if still pushed, so + // the following push finds it gone again. + if (gone.length > 0) { + entry.delete_chats = gone.slice(0, MAX_DELETES_PER_ENTRY) + for (const id of gone.slice(MAX_DELETES_PER_ENTRY)) { + next.chats[id] = prev.chats[id] + carried = true + } + changed = true + } + if (evicted.length > 0) { + entry.delete_images = evicted + .slice(0, MAX_DELETES_PER_ENTRY) + .map(([id, chatId]) => ({ chat_id: chatId, id })) + for (const [id, chatId] of evicted.slice(MAX_DELETES_PER_ENTRY)) { + next.images[id] = chatId + carried = true + } + changed = true + } + } + + const fingerprint = artifactsFingerprint(artifacts) + next.artifacts = fingerprint + if (prev?.artifacts !== fingerprint && (artifacts.items.length > 0 || prev?.artifacts)) { + entry.artifacts = { items: artifacts.items, versions: artifacts.versions } + changed = true + } + + return { + workspaceId, + entry: changed ? entry : undefined, + images, + removeFrom, + next, + carried, + whole: prev === undefined + } +} + +/** Bytes a JSON body would carry for this value, as sent: UTF-8, not UTF-16 code units, + * which would under-count a transcript in a non-Latin script by up to three times. Counted + * rather than encoded: the values measured are the multi-megabyte ones. */ +export function jsonBytes(value: unknown): number { + const text = JSON.stringify(value) + let bytes = 0 + for (let i = 0; i < text.length; i++) { + const c = text.charCodeAt(i) + if (c < 0x80) bytes += 1 + else if (c < 0x800) bytes += 2 + else if (c >= 0xd800 && c <= 0xdbff) { + // A surrogate pair is one four-byte code point. + bytes += 4 + i++ + } else bytes += 3 + } + return bytes +} + +/** Object-store calls the server makes for an entry, the unit its per-request cap counts. */ +export function operationsOf(entry: AISessionBackupPush): number { + return ( + (entry.chats?.length ?? 0) + + (entry.images?.length ?? 0) + + (entry.delete_chats?.length ?? 0) + + (entry.delete_images?.length ?? 0) + ) +} + +export interface PushBody { + owner: string + sessions: AISessionBackupPush[] + removed?: string[] +} + +/** The server's caps on chats and on each delete list per entry. */ +export const MAX_CHATS_PER_ENTRY = 100 +export const MAX_DELETES_PER_ENTRY = 1000 + +/** + * Break an entry that outgrows the target, or the server's per-entry chat cap, into + * chat-only entries, each written on its own, with everything else riding on the last one: + * the entries go out in order and the server lists the session by the last, so the marker + * never lists a chat that has not landed. (A push of the session whole moves the head to + * whichever part goes first; see the mirror.) + */ +export function splitEntry(entry: AISessionBackupPush, targetBytes: number): AISessionBackupPush[] { + if ( + !entry.chats || + entry.chats.length <= 1 || + (entry.chats.length <= MAX_CHATS_PER_ENTRY && jsonBytes(entry) <= targetBytes) + ) { + return [entry] + } + const { chats, ...rest } = entry + const parts: AISessionBackupPush[] = [] + let current: typeof chats = [] + let size = 0 + for (const chat of chats) { + const bytes = jsonBytes(chat) + if ( + current.length > 0 && + (current.length >= MAX_CHATS_PER_ENTRY || size + bytes > targetBytes) + ) { + parts.push({ id: entry.id, chats: current }) + current = [] + size = 0 + } + current.push(chat) + size += bytes + } + parts.push({ ...rest, chats: current }) + return parts +} diff --git a/frontend/src/lib/components/sessions/sessionMirrorSignal.ts b/frontend/src/lib/components/sessions/sessionMirrorSignal.ts new file mode 100644 index 0000000000..eb936d2e22 --- /dev/null +++ b/frontend/src/lib/components/sessions/sessionMirrorSignal.ts @@ -0,0 +1,42 @@ +// The one-way channel from the IndexedDB write funnels (session records, chat history, +// artifacts) to the session backup. Import-free on purpose: the stores it is called from +// must not depend on the backup module, which depends on all of them. + +// `email` names the user whose store the write landed in (from the store's scoped name): +// the current user may have changed while the write was pending, and the mark belongs to +// the store's user, not to whoever is logged in when it completes. +export type MirrorSignal = + | { kind: 'dirty'; sessionId: string; chatId?: string; email?: string } + | { kind: 'removed'; sessionId: string; workspaceId?: string; email?: string } + +let handler: ((signal: MirrorSignal) => void) | undefined +// Signals raised before the backup module registered, replayed to it on registration. +let buffered: MirrorSignal[] = [] + +function emit(signal: MirrorSignal): void { + if (handler) handler(signal) + else buffered.push(signal) +} + +/** A durable local write landed for this session (and, when known, this chat) in the + * store of `email`. */ +export function markSessionDirty(sessionId: string, chatId?: string, email?: string): void { + emit({ kind: 'dirty', sessionId, chatId, email }) +} + +/** The user deleted this session; its backup goes with it. */ +export function markSessionRemoved(sessionId: string, workspaceId?: string, email?: string): void { + emit({ kind: 'removed', sessionId, workspaceId, email }) +} + +export function onMirrorSignal(fn: (signal: MirrorSignal) => void): void { + handler = fn + const replay = buffered + buffered = [] + for (const signal of replay) fn(signal) +} + +export function __resetMirrorSignalForTesting(): void { + handler = undefined + buffered = [] +} diff --git a/frontend/src/lib/components/sessions/sessionState.svelte.ts b/frontend/src/lib/components/sessions/sessionState.svelte.ts index d19faeac92..2670b3fa4d 100644 --- a/frontend/src/lib/components/sessions/sessionState.svelte.ts +++ b/frontend/src/lib/components/sessions/sessionState.svelte.ts @@ -24,8 +24,10 @@ import { workspaceRootId } from './sessionScope.svelte' import { clearSessionRecovered } from './sessionRecoveryNotice.svelte' import { type DBSchema, type IDBPDatabase } from 'idb' import { userScopedDb } from '$lib/userScopedDb' +import { emailOfScopedKey, scopedKeyFor } from '$lib/userScopedStorage' import { deleteItemsForSession } from '../copilot/chat/files/attachedFilesDB' import { deleteArtifactsForSession } from '../copilot/chat/artifacts/artifactsDB' +import { markSessionDirty, markSessionRemoved } from './sessionMirrorSignal' // Switch the global workspace iff the target differs from the active one // and is non-empty. Centralises the "session needs its workspace in focus" @@ -102,6 +104,10 @@ export type Session = { // archived (not by the user). Lets reconciliation auto-unarchive the session // when the workspace is unarchived, while leaving user-archived sessions be. archivedByWorkspace?: boolean + // How many times the session moved to another workspace. The backup keeps it + // with the session's marker, so a restore that finds a copy in two workspaces + // (moved, the old copy not yet removed) takes the later one without a clock. + moves?: number // In-memory-only flag: the session exists but hasn't been written to // IndexedDB yet. Set at creation, cleared on the first genuine user touch // (typed prompt, workspace/fork pick, preview tab, rename) which persists @@ -444,6 +450,7 @@ async function deleteSessionRow(db: IDBPDatabase, id: string): Pr async function putSessionRow(db: IDBPDatabase, s: Session): Promise { if (deletedSessionIds.has(s.id)) return await db.put('sessions', s) + markSessionDirty(s.id, undefined, emailOfScopedKey(SESSIONS_DB, db.name)) } // Write-behind a single session record. Transient sessions are in-memory only @@ -775,6 +782,15 @@ export function findEmptyLandingSession(): Session | undefined { ) } +// Session names are a per-browser counter (`session-N`) that the sessions page puts in +// its URL, so a new or restored record takes the number after the highest in use. +function nextSessionNumber(sessions: Session[]): number { + const numbers = sessions + .map((s) => /^session-(\d+)$/.exec(s.name)?.[1]) + .map((n) => (n ? parseInt(n, 10) : 0)) + return (numbers.length ? Math.max(...numbers) : 0) + 1 +} + export function createSession(): Session { // Reuse an existing untouched draft from the active family rather than pile a // blank entry on every `+`, so several pending sessions can still be built up @@ -795,10 +811,7 @@ export function createSession(): Session { return reusable } sessionState.sessions = sessionState.sessions.filter((s) => !isDiscardableDraft(s)) - const existingNumbers = sessionState.sessions - .map((s) => /^session-(\d+)$/.exec(s.name)?.[1]) - .map((n) => (n ? parseInt(n, 10) : 0)) - const next = (existingNumbers.length ? Math.max(...existingNumbers) : 0) + 1 + const next = nextSessionNumber(sessionState.sessions) // Start in the workspace you're in. The one exception: a root you can't // deploy to (locked, no bypass) steers to its dev, since a session there // couldn't edit anything. The picker lets you switch. @@ -1069,6 +1082,7 @@ export async function moveSessionToWorkspace(id: string, newWorkspaceId: string) const s = sessionState.sessions.find((x) => x.id === id) if (!s) return if (s.workspace_id === newWorkspaceId) return + if (s.workspace_id !== undefined) s.moves = (s.moves ?? 0) + 1 s.workspace_id = newWorkspaceId delete s.pending_workspace_id delete s.pending_fork @@ -1144,9 +1158,63 @@ export function deleteSession(id: string) { // GC any linked files and artifacts persisted for this session. void deleteItemsForSession(id) void deleteArtifactsForSession(id) + // Only a delete the user asked for takes the backup with it: the workspace-lifecycle + // removals above keep theirs, so a session dropped by a wrong reconcile can be restored. + markSessionRemoved(id, s.workspace_id) logFeatureUsage('ai_session', 'deleted', { entityId: id, workspace: s.workspace_id }) } +// --- Session backup support (sessionMirror) --- + +export function isSessionTombstoned(id: string): boolean { + return deletedSessionIds.has(id) +} + +// Every stored record of the named user, or undefined when the store is unavailable or +// already serves someone else: the backup captures its user up front and must not follow +// an in-place account switch. +export async function readStoredSessions(email: string): Promise { + if (!BROWSER) return undefined + const db = await sessionsDb.whenReady() + if (!db || db.name !== scopedKeyFor(SESSIONS_DB, email)) return undefined + try { + return await db.getAll('sessions') + } catch (e) { + console.error('Failed to read sessions from IndexedDB', e) + return undefined + } +} + +// Add restored records for sessions this browser does not have, and re-hydrate the list. +// A record that exists, or was deleted here, is left alone: the local copy is the newer +// one. Returns the ids written. +export async function importSessions(records: Session[], email: string): Promise { + if (!BROWSER) return [] + const db = await sessionsDb.whenReady() + if (!db || db.name !== scopedKeyFor(SESSIONS_DB, email)) return [] + const imported: string[] = [] + try { + const tx = db.transaction('sessions', 'readwrite') + const existing = new Set((await tx.store.getAllKeys()).map(String)) + let next = nextSessionNumber([...(await tx.store.getAll()), ...sessionState.sessions]) + for (const r of records) { + if (existing.has(r.id) || deletedSessionIds.has(r.id)) continue + const record: Session = { ...r, name: `session-${next++}` } + delete record.transient + delete record.workspace_root_id + ensureSessionRootId(record) + await tx.store.put(record) + imported.push(record.id) + } + await tx.done + } catch (e) { + console.error('Failed to import sessions', e) + return [] + } + if (imported.length > 0) await hydrateSessions() + return imported +} + export function setSessionChatId(sessionId: string, chatId: string) { const s = sessionState.sessions.find((x) => x.id === sessionId) if (s && s.chatId !== chatId) { @@ -1172,6 +1240,7 @@ async function patchStoredSessionChatId(s: Session, chatId: string): Promise { userStore.set(undefined) await vi.waitFor(() => expect(sessionState.sessions).toEqual([])) }) + + // A restored backup must never replace what this browser has, come back after the + // user deleted it here, or take a name the sessions page already routes by. + it('importSessions adds only unknown, undeleted records under fresh names', async () => { + const user = freshUser() + await login(user) + const local = session({ id: 'local', name: 'session-3', createdAt: 1, summary: 'mine' }) + await putSession(local) + sessionState.sessions.push(local) + deleteSession('local') + await flush() + await putSession(session({ id: 'kept', name: 'session-5', createdAt: 2, summary: 'kept' })) + + const imported = await importSessions( + [ + session({ id: 'local', name: 'session-1', createdAt: 1, summary: 'remote copy' }), + session({ id: 'kept', name: 'session-1', createdAt: 2, summary: 'remote copy' }), + session({ id: 'new', name: 'session-1', createdAt: 3, workspace_id: 'ws' }) + ], + user.email + ) + expect(imported).toEqual(['new']) + await vi.waitFor(() => + expect(sessionState.sessions.map((s) => [s.id, s.name])).toEqual([ + ['new', 'session-6'], + ['kept', 'session-5'] + ]) + ) + expect(sessionState.sessions.find((s) => s.id === 'kept')?.summary).toBe('kept') + + // The wrong user's name gets nothing written. + expect(await importSessions([session({ id: 'other', createdAt: 4 })], 'nobody@x')).toEqual([]) + }) }) diff --git a/frontend/src/lib/components/workspaceSettings/AISettings.svelte b/frontend/src/lib/components/workspaceSettings/AISettings.svelte index ae9d2640b0..7e8b9cc76e 100644 --- a/frontend/src/lib/components/workspaceSettings/AISettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/AISettings.svelte @@ -28,6 +28,7 @@ import ModelPricing from './ModelPricing.svelte' import AiUsagePanel from './AiUsagePanel.svelte' import { setCopilotInfo } from '$lib/aiStore' + import { backupSettingsChanged } from '$lib/components/sessions/sessionMirror.svelte' import AIPromptsModal from '../settings/AIPromptsModal.svelte' import { Settings } from 'lucide-svelte' import { untrack } from 'svelte' @@ -79,6 +80,7 @@ let usingOpenaiClientCredentialsOauth = $state(false) let workspaceOverrideEditorOpened = $state(false) let copilotDisabled = $state(false) + let sessionsStorageDisabled = $state(false) // --- Initial state for dirty tracking --- let initialAiProviders: Exclude = $state({}) @@ -90,6 +92,7 @@ let initialModelPricing: Record = $state({}) let initialPrompts: Record = $state({}) let initialCopilotDisabled = $state(false) + let initialSessionsStorageDisabled = $state(false) let lastLoadedConfigKey = $state(undefined) function clone(v: T): T { @@ -118,6 +121,7 @@ maxTokensPerModel = clone(config?.max_tokens_per_model ?? {}) modelPricing = clone(config?.model_pricing ?? {}) copilotDisabled = config?.copilot_disabled === true + sessionsStorageDisabled = config?.sessions_storage_disabled === true for (const mode of ['edit', 'fix', 'gen']) { if (!(mode in customPrompts)) { customPrompts[mode] = '' @@ -135,6 +139,7 @@ initialModelPricing = clone(modelPricing) initialPrompts = clone(customPrompts) initialCopilotDisabled = copilotDisabled + initialSessionsStorageDisabled = sessionsStorageDisabled } export function loadFromConfig(config: AIConfig | undefined) { @@ -151,6 +156,7 @@ maxTokensPerModel = clone(initialMaxTokensPerModel) modelPricing = clone(initialModelPricing) copilotDisabled = initialCopilotDisabled + sessionsStorageDisabled = initialSessionsStorageDisabled } $effect(() => { @@ -186,7 +192,8 @@ JSON.stringify(customPrompts) !== JSON.stringify(initialCustomPrompts) || JSON.stringify(maxTokensPerModel) !== JSON.stringify(initialMaxTokensPerModel) || JSON.stringify(modelPricing) !== JSON.stringify(initialModelPricing) || - copilotDisabled !== initialCopilotDisabled + copilotDisabled !== initialCopilotDisabled || + sessionsStorageDisabled !== initialSessionsStorageDisabled ) $effect(() => { @@ -291,8 +298,9 @@ .filter(([_, prompt]) => prompt.trim().length > 0) .reduce((acc, [mode, prompt]) => ({ ...acc, [mode]: prompt }), {}) - // The flag is the one thing a workspace on instance defaults still stores of its own. + // The flags are what a workspace on instance defaults still stores of its own. const copilot_disabled = copilotDisabled ? true : undefined + const sessions_storage_disabled = sessionsStorageDisabled ? true : undefined return Object.keys(aiProviders ?? {}).length > 0 ? { providers: aiProviders, @@ -303,9 +311,10 @@ max_tokens_per_model: Object.keys(maxTokensPerModel).length > 0 ? maxTokensPerModel : undefined, model_pricing: Object.keys(modelPricing).length > 0 ? modelPricing : undefined, - copilot_disabled + copilot_disabled, + sessions_storage_disabled } - : { copilot_disabled } + : { copilot_disabled, sessions_storage_disabled } } function isSaveDisabled(): boolean { @@ -332,6 +341,7 @@ async function editCopilotConfig(): Promise { const config = buildConfig() + const backupsToggled = sessionsStorageDisabled !== initialSessionsStorageDisabled let settingsState: GetCopilotSettingsStateResponse | undefined if (customSave) { @@ -348,6 +358,9 @@ instance_ai_summary: response.instance_ai_summary } sendUserToast('AI settings updated') + // This page's session backups follow the switch at once, rather than at the + // next page load. + if (backupsToggled) backupSettingsChanged(effectiveWorkspace) } storeInitialState() // Hand the parent what was persisted: it owns `initialConfig`, and this component is @@ -646,6 +659,18 @@ options={{ right: 'Hide AI sessions in this workspace' }} /> + + { + sessionsStorageDisabled = e.detail + }} + options={{ right: 'Do not back AI sessions up to the workspace storage' }} + /> + {/if} diff --git a/frontend/src/lib/userScopedDb.ts b/frontend/src/lib/userScopedDb.ts index 71d80a369f..d1283b2c73 100644 --- a/frontend/src/lib/userScopedDb.ts +++ b/frontend/src/lib/userScopedDb.ts @@ -1,4 +1,11 @@ -import { openDB as idbOpenDB, deleteDB as idbDeleteDB, type DBSchema, type IDBPDatabase } from 'idb' +import { + openDB as idbOpenDB, + deleteDB as idbDeleteDB, + type DBSchema, + type IDBPDatabase, + type IDBPTransaction, + type StoreNames +} from 'idb' import { scopedKey } from '$lib/userScopedStorage' // Per-user IndexedDB lifecycle, shared by the session list and the copilot @@ -21,7 +28,12 @@ export interface UserScopedDbMigrateDeps { export interface UserScopedDbOptions { version: number - upgrade: (db: IDBPDatabase) => void + // The version-change transaction is the only way to add an index to a store that + // already exists; a store being created gets it from the store handle instead. + upgrade: ( + db: IDBPDatabase, + tx: IDBPTransaction[], 'versionchange'> + ) => void // Invoked once per scoped name right after a successful open. The fn owns its // own "already migrated / not applicable" gate (e.g. checking a store's // count) — claim-then-delete legacy data lives here. @@ -100,11 +112,11 @@ export function userScopedDb( try { let handle: IDBPDatabase | undefined const db = await openDB(name, opts.version, { - upgrade(database) { + upgrade(database, _oldVersion, _newVersion, transaction) { // The version-change transaction is ours: nothing is queued ahead of this // open any more, and what remains is our own upgrade running. stopWaiting() - opts.upgrade(database) + opts.upgrade(database, transaction) }, // Another tab is opening this database at a higher version, which our open // connection would block indefinitely. Let go so their upgrade lands; this diff --git a/frontend/src/lib/userScopedStorage.ts b/frontend/src/lib/userScopedStorage.ts index 905d609b61..e3ee1d842d 100644 --- a/frontend/src/lib/userScopedStorage.ts +++ b/frontend/src/lib/userScopedStorage.ts @@ -52,7 +52,19 @@ export function getCurrentUserEmail(): string | undefined { // treat that as "do not read/write" so we never touch a browser-global key. export function scopedKey(base: string): string | undefined { if (!currentEmail) return undefined - return `${base}::${currentEmail}` + return scopedKeyFor(base, currentEmail) +} + +// The key a base name has for a given user, for work that captured its user up front and +// must not follow an in-place account switch (the session backup flush). +export function scopedKeyFor(base: string, email: string): string { + return `${base}::${email}` +} + +// The email a scoped key or database name was built for, so a write that landed in a +// store can name the user it belongs to even after the current user changed. +export function emailOfScopedKey(base: string, key: string): string | undefined { + return key.startsWith(`${base}::`) ? key.slice(base.length + 2) : undefined } // Register a callback invoked whenever the scoping email changes. Fired once diff --git a/frontend/src/routes/(root)/(logged)/+layout.svelte b/frontend/src/routes/(root)/(logged)/+layout.svelte index 4447a8d4b8..225b9443d3 100644 --- a/frontend/src/routes/(root)/(logged)/+layout.svelte +++ b/frontend/src/routes/(root)/(logged)/+layout.svelte @@ -92,6 +92,7 @@ import { parsePreviewItemRoute } from '$lib/components/sessions/previewPaths' import { rememberNavRoute } from '$lib/components/sessions/sessionSwitch.svelte' import { sessionState } from '$lib/components/sessions/sessionState.svelte' + import { restoreSessionBackups } from '$lib/components/sessions/sessionMirror.svelte' import { currentWorkspaceRootId } from '$lib/components/sessions/sessionScope.svelte' import WorkspaceScopeHeader from '$lib/components/sidebar/WorkspaceScopeHeader.svelte' import { DEFAULT_HUB_BASE_URL } from '$lib/hub' @@ -725,6 +726,16 @@ $workspaceStore untrack(() => updateUserStore($workspaceStore)) }) + // Bring back the AI sessions this browser lacks for the workspace family in view, once + // the local list is known (so nothing it has is fetched again) and the memberships have + // resolved (the family is derived from them). + $effect(() => { + const ws = $workspaceStore + const ready = sessionState.hydrated && $usersWorkspaceStore !== undefined + if (globalAiEnabled && ready && ws && !$userStore?.operator) { + untrack(() => restoreSessionBackups(ws)) + } + }) // While a fork is reachable, mirror its parent linkage to localStorage so a // later reload landing on a now-deleted fork can return to the parent (see // forkParentMemory + the deleted-fork recovery in the root layout). From 1c17b3c8dbc43b26adc0a2d4e738cacff8a54e8d Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 15 Sep 2026 22:18:37 +0200 Subject: [PATCH 132/144] test: pin unlisting on a failed multi-object ai session push (#11150) * test: pin that a failed multi-object incremental push leaves the session unlisted Co-Authored-By: Claude Fable 5.1 * test: pick the newest backup generation in the ai sessions test helper Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Fable 5.1 --- backend/tests/ai_sessions.rs | 107 ++++++++++++++++++++++++++++++++++- 1 file changed, 104 insertions(+), 3 deletions(-) diff --git a/backend/tests/ai_sessions.rs b/backend/tests/ai_sessions.rs index f76385d4e1..340a1380da 100644 --- a/backend/tests/ai_sessions.rs +++ b/backend/tests/ai_sessions.rs @@ -87,7 +87,8 @@ async fn rotate(base: &str, key: &str) -> anyhow::Result<()> { } /// The user's prefix on disk, `windmill_ai_sessions/{w_id}/g{generation}/{email hash}`, -/// under whichever generation is current. +/// under the newest generation: deleting an older generation's objects leaves its +/// directories behind, and `read_dir` order differs across filesystems. fn user_root(storage_dir: &std::path::Path, email: &str) -> std::path::PathBuf { let workspace = storage_dir.join("windmill_ai_sessions/test-workspace"); let hash = calculate_hash(email); @@ -96,8 +97,18 @@ fn user_root(storage_dir: &std::path::Path, email: &str) -> std::path::PathBuf { .into_iter() .flatten() .flatten() - .map(|entry| entry.path().join(&hash)) - .find(|path| path.exists()) + .filter_map(|entry| { + let generation: i64 = entry + .file_name() + .to_str()? + .strip_prefix('g')? + .parse() + .ok()?; + Some((generation, entry.path().join(&hash))) + }) + .filter(|(_, path)| path.exists()) + .max_by_key(|(generation, _)| *generation) + .map(|(_, path)| path) .expect("the user has backups under the current key") } @@ -945,6 +956,96 @@ async fn test_backups_round_trip_encrypted_and_scoped_to_the_user( .await?; assert_eq!(resp.status(), 200); + // An incremental part changing more than one object unlists the session before its + // writes, so one write failing after another landed leaves it absent rather than listed + // as a mix of old and new pieces. A directory planted at `artifacts.json` fails that write. + let s10_head = + json!({ "id": "s10", "workspace_id": "test-workspace", "createdAt": 10, "chatId": "c1" }); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s10", "whole": true, "head": s10_head, "chats": s9_chats(&["c1"]), "artifacts": { "items": ["a1"] } }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let s10_dir = user_root(storage_dir.path(), "test@windmill.dev").join("sessions/s10"); + let artifacts_path = s10_dir.join("artifacts.json"); + std::fs::remove_file(&artifacts_path)?; + std::fs::create_dir(&artifacts_path)?; + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s10", "chats": s9_chats(&["c2"]), "artifacts": { "items": ["a2"] } }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let answer: Value = resp.json().await?; + assert!( + answer["results"][0]["error"].is_string(), + "the artifacts write must fail: {answer}" + ); + assert!(answer["results"][0]["needs_whole"].is_null()); + assert!( + s10_dir.join("chats/c2.json").is_file(), + "the chat landed before the artifacts failed" + ); + let s10_listed = |listing: Value| { + listing["sessions"] + .as_array() + .unwrap() + .iter() + .any(|s| s["id"] == "s10") + }; + assert!(!s10_listed(list(&base, "SECRET_TOKEN").await?)); + assert_eq!( + pull(&base, "SECRET_TOKEN", &["s10"]).await?["sessions"], + json!([]) + ); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s10", "chats": s9_chats(&["c3"]) }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let answer: Value = resp.json().await?; + assert_eq!(answer["results"][0]["needs_whole"], true); + assert!(!s10_listed(list(&base, "SECRET_TOKEN").await?)); + std::fs::remove_dir(&artifacts_path)?; + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s10", "whole": true, "head": s10_head, "chats": s9_chats(&["c1", "c2", "c3"]), "artifacts": { "items": ["a2"] } }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + assert!(s10_listed(list(&base, "SECRET_TOKEN").await?)); + let pulled = pull(&base, "SECRET_TOKEN", &["s10"]).await?; + assert_eq!( + pulled["sessions"][0]["artifacts"], + json!({ "items": ["a2"] }) + ); + assert_eq!(pulled_chats(pulled), vec!["c1", "c2", "c3"]); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "removed": ["s10"] }), + ) + .await?; + assert_eq!(resp.status(), 200); + // Removal empties both prefixes. let resp = push( &base, From f082fddf419218e880371f28abe477c64bd7394a Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 15 Sep 2026 23:09:45 +0200 Subject: [PATCH 133/144] [ee] feat: fall back to instance storage for AI session backups (#11153) * feat: instance object store as fallback for AI session backups Co-Authored-By: Claude Fable 5.1 * fix: fence the instance store sweep by generation, name it by location Co-Authored-By: Claude Opus 5 (1M context) * test: pin that an instance store location tells endpoints apart Co-Authored-By: Claude Opus 5 (1M context) * fix: show the instance storage fallback setting on while it is unset Co-Authored-By: Claude Opus 5 (1M context) * fix: check the generation fence queries at compile time Co-Authored-By: Claude Opus 5 (1M context) * fix: stop the instance storage fallback once the plan is Pro Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Fable 5.1 --- ...7f9ed1c0f4b4d1f07789b40945cfb8a3a7b39.json | 14 ++ ...e67c90e86d35dd5f74b1366f802dbb951ef9a.json | 22 ++ ...cb6a994e969d915d50afbf0cb8547e6decf11.json | 22 ++ backend/ee-repo-ref.txt | 2 +- backend/tests/ai_sessions.rs | 226 ++++++++++++++++- .../src/ai_session_backups.rs | 231 ++++++++++++++---- .../windmill-api-workspaces/src/workspaces.rs | 26 ++ backend/windmill-api/openapi.yaml | 7 + backend/windmill-api/src/ai_sessions.rs | 85 +++++-- .../windmill-common/src/global_settings.rs | 5 + .../windmill-common/src/instance_config.rs | 2 + backend/windmill-object-store/src/lib.rs | 155 +++++++++--- docs/ai-session-backups.md | 52 +++- .../lib/components/InstanceSettings.svelte | 8 +- .../src/lib/components/instanceSettings.ts | 11 + .../sessions/sessionMirror.svelte.ts | 31 ++- .../components/sessions/sessionMirror.test.ts | 71 ++++++ .../components/sessions/sessionMirrorPlan.ts | 34 ++- .../workspaceSettings/AISettings.svelte | 2 +- 19 files changed, 871 insertions(+), 135 deletions(-) create mode 100644 backend/.sqlx/query-c2f3492c2d80f5c6d157d8c1dab7f9ed1c0f4b4d1f07789b40945cfb8a3a7b39.json create mode 100644 backend/.sqlx/query-dfa82a3f291cdc8f05cc4af6114e67c90e86d35dd5f74b1366f802dbb951ef9a.json create mode 100644 backend/.sqlx/query-ea7bc2e5f53144ca23f8b3dea71cb6a994e969d915d50afbf0cb8547e6decf11.json diff --git a/backend/.sqlx/query-c2f3492c2d80f5c6d157d8c1dab7f9ed1c0f4b4d1f07789b40945cfb8a3a7b39.json b/backend/.sqlx/query-c2f3492c2d80f5c6d157d8c1dab7f9ed1c0f4b4d1f07789b40945cfb8a3a7b39.json new file mode 100644 index 0000000000..9724a99b18 --- /dev/null +++ b/backend/.sqlx/query-c2f3492c2d80f5c6d157d8c1dab7f9ed1c0f4b4d1f07789b40945cfb8a3a7b39.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings SET ai_sessions_backup_generation = ai_sessions_backup_generation + 1 WHERE workspace_id = $1 AND large_file_storage IS NULL", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "c2f3492c2d80f5c6d157d8c1dab7f9ed1c0f4b4d1f07789b40945cfb8a3a7b39" +} diff --git a/backend/.sqlx/query-dfa82a3f291cdc8f05cc4af6114e67c90e86d35dd5f74b1366f802dbb951ef9a.json b/backend/.sqlx/query-dfa82a3f291cdc8f05cc4af6114e67c90e86d35dd5f74b1366f802dbb951ef9a.json new file mode 100644 index 0000000000..0b17793580 --- /dev/null +++ b/backend/.sqlx/query-dfa82a3f291cdc8f05cc4af6114e67c90e86d35dd5f74b1366f802dbb951ef9a.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT large_file_storage IS NOT NULL AS \"has_storage!\" FROM workspace_settings WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "has_storage!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "dfa82a3f291cdc8f05cc4af6114e67c90e86d35dd5f74b1366f802dbb951ef9a" +} diff --git a/backend/.sqlx/query-ea7bc2e5f53144ca23f8b3dea71cb6a994e969d915d50afbf0cb8547e6decf11.json b/backend/.sqlx/query-ea7bc2e5f53144ca23f8b3dea71cb6a994e969d915d50afbf0cb8547e6decf11.json new file mode 100644 index 0000000000..a344e64392 --- /dev/null +++ b/backend/.sqlx/query-ea7bc2e5f53144ca23f8b3dea71cb6a994e969d915d50afbf0cb8547e6decf11.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT ai_sessions_backup_generation FROM workspace_settings WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "ai_sessions_backup_generation", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "ea7bc2e5f53144ca23f8b3dea71cb6a994e969d915d50afbf0cb8547e6decf11" +} diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 7acd297dac..5d28f73b5a 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -1c1dab33563c4907aff8b0da825fb66db60af82a +93433d7c9dc34f2c0f56a5297d3453aabd2f9472 \ No newline at end of file diff --git a/backend/tests/ai_sessions.rs b/backend/tests/ai_sessions.rs index 340a1380da..3eb28cfae8 100644 --- a/backend/tests/ai_sessions.rs +++ b/backend/tests/ai_sessions.rs @@ -34,6 +34,54 @@ async fn configure_primary_lfs(db: &Pool, root_path: &str) -> anyhow:: Ok(()) } +/// Configures the primary storage through the route, which is what sweeps the workspace's +/// backups out of the instance store. +async fn configure_primary_lfs_via_route(base: &str, root_path: &str) -> anyhow::Result<()> { + let resp = authed( + client().post(format!("{base}/workspaces/edit_large_file_storage_config")), + "SECRET_TOKEN", + ) + .json(&json!({ "large_file_storage": { + "type": "FilesystemStorage", + "root_path": root_path, + "public_resource": false, + "advanced_permissions": null, + "secondary_storage": {} + }})) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + Ok(()) +} + +/// The instance setting allowing the instance store to stand in for a workspace without +/// storage: `None` leaves it unset, which is on. +async fn set_instance_fallback(db: &Pool, on: Option) -> anyhow::Result<()> { + sqlx::query("DELETE FROM global_settings WHERE name = 'ai_sessions_instance_storage_fallback'") + .execute(db) + .await?; + if let Some(on) = on { + sqlx::query( + "INSERT INTO global_settings (name, value) VALUES ('ai_sessions_instance_storage_fallback', $1)", + ) + .bind(json!(on)) + .execute(db) + .await?; + } + Ok(()) +} + +/// Polls until nothing is under the directory, for a deletion that runs off the request. +async fn wait_until_empty(dir: &std::path::Path, what: &str) { + for _ in 0..100 { + if files_under(dir).is_empty() { + return; + } + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + panic!("{what}: objects left under {}", dir.display()); +} + async fn list(base: &str, token: &str) -> anyhow::Result { let resp = authed(client().get(format!("{base}/ai/sessions/list")), token) .send() @@ -143,7 +191,9 @@ async fn test_backups_round_trip_encrypted_and_scoped_to_the_user( server.addr.port() ); - // No storage configured: the browser is told to stop trying. + // No storage configured, and the instance store (another test of this process may + // have loaded one) not allowed to stand in: the browser is told to stop trying. + set_instance_fallback(&db, Some(false)).await?; let listing = list(&base, "SECRET_TOKEN").await?; assert_eq!(listing["enabled"], false); assert_eq!(listing["sessions"], json!([])); @@ -1184,3 +1234,177 @@ async fn test_backup_writes_are_refused_for_the_wrong_owner_token_or_id( assert!(files_under(storage_dir.path()).is_empty()); Ok(()) } + +/// Puts the process-wide instance store back to none, even when an assertion fails. +struct ResetInstanceStore; +impl Drop for ResetInstanceStore { + fn drop(&mut self) { + if let Ok(mut store) = windmill_object_store::OBJECT_STORE_SETTINGS.try_write() { + *store = None; + } + } +} + +/// Puts the process-wide license key id back to none, an Enterprise plan in this build, +/// even when an assertion fails. +struct ResetLicensePlan; +impl Drop for ResetLicensePlan { + fn drop(&mut self) { + windmill_common::ee::LICENSE_KEY_ID.store(std::sync::Arc::new(String::new())); + } +} + +/// A workspace without storage of its own backs up to the instance object store, every +/// answer saying so (`fallback`); a storage of its own, once configured, answers instead, +/// under a generation past everything the workspace left in the instance store, which the +/// change deletes; a plan switched to Pro stops the fallback with the store still loaded. +#[sqlx::test(fixtures("base"))] +async fn test_backups_fall_back_to_the_instance_storage(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let base = format!( + "http://localhost:{}/api/w/test-workspace", + server.addr.port() + ); + + // The instance store, built from its settings as `reload_object_store_setting` does. + let instance_dir = tempfile::tempdir()?; + let instance_root = instance_dir.path().to_string_lossy().to_string(); + *windmill_object_store::OBJECT_STORE_SETTINGS.write().await = Some( + windmill_object_store::build_object_store_from_settings( + windmill_object_store::ObjectSettings::Filesystem( + windmill_object_store::FilesystemSettings { root_path: instance_root.clone() }, + ), + None, + ) + .await?, + ); + let _reset = ResetInstanceStore; + let in_instance = instance_dir + .path() + .join("windmill_ai_sessions/test-workspace"); + + // Turned off by the instance setting: the browser is told to stop trying. + set_instance_fallback(&db, Some(false)).await?; + assert_eq!(list(&base, "SECRET_TOKEN").await?["enabled"], false); + set_instance_fallback(&db, None).await?; + + // On, as it is unless turned off: the backups land in the instance store, under the + // workspace's prefix, and every answer says which kind of store it came from. + let head = + json!({ "id": "s1", "workspace_id": "test-workspace", "createdAt": 1, "chatId": "c1" }); + let entry = json!({ "id": "s1", "whole": true, "head": head, "chats": [{ "id": "c1", "record": { "id": "c1" } }] }); + let push_whole = || { + push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "sessions": [entry.clone()] }), + ) + }; + let resp = push_whole().await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let pushed: Value = resp.json().await?; + assert_eq!(pushed["fallback"], true); + assert_eq!(pushed["results"], json!([{ "id": "s1" }])); + let listing = list(&base, "SECRET_TOKEN").await?; + assert_eq!(listing["enabled"], true); + assert_eq!(listing["fallback"], true); + assert_eq!(listing["sessions"][0]["id"], "s1"); + let fallback_storage_id = listing["storage_id"].clone(); + let pulled = pull(&base, "SECRET_TOKEN", &["s1"]).await?; + assert_eq!(pulled["fallback"], true); + assert_eq!(pulled["sessions"][0]["head"], head); + assert!(!files_under(&in_instance).is_empty()); + + // The workspace's storage usage counts them, under a name of their own. + let resp = authed( + client().get(format!("{base}/job_helpers/storage_usage")), + "SECRET_TOKEN", + ) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let usage: Value = resp.json().await?; + let fallback_usage = usage["storages"] + .as_array() + .unwrap() + .iter() + .find(|s| s["storage"] == "_ai_sessions_fallback_") + .unwrap_or_else(|| panic!("no fallback usage in {usage}")); + assert!(fallback_usage["bytes"].as_i64().unwrap() > 0); + + // A key rotation sweeps the older generation out of the instance store too. + rotate(&base, &"c".repeat(64)).await?; + wait_until_empty(&in_instance, "a rotation on the instance store").await; + assert_eq!(list(&base, "SECRET_TOKEN").await?["sessions"], json!([])); + assert_eq!(push_whole().await?.status(), 200); + assert!(!files_under(&in_instance).is_empty()); + + // A storage of its own answers instead, under a generation the configuration moved past + // everything the workspace left in the instance store: nothing there is read again, + // whichever store a later return to the fallback finds, and it is deleted. + let before = list(&base, "SECRET_TOKEN").await?; + let storage_dir = tempfile::tempdir()?; + configure_primary_lfs_via_route(&base, &storage_dir.path().to_string_lossy()).await?; + wait_until_empty(&in_instance, "configuring a workspace storage").await; + let listing = list(&base, "SECRET_TOKEN").await?; + assert_eq!(listing["enabled"], true); + assert!(listing.get("fallback").is_none(), "{listing}"); + assert_ne!(listing["storage_id"], fallback_storage_id); + assert_eq!( + listing["backup_generation"].as_i64(), + before["backup_generation"].as_i64().map(|g| g + 1), + "configuring a storage over the fallback must move the generation on" + ); + assert_eq!(listing["sessions"], json!([])); + assert_eq!(push_whole().await?.status(), 200); + assert!(!files_under(storage_dir.path()).is_empty()); + assert!(files_under(&in_instance).is_empty()); + let resp = authed( + client().get(format!("{base}/job_helpers/storage_usage?refresh=true")), + "SECRET_TOKEN", + ) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let usage: Value = resp.json().await?; + assert!( + !usage.to_string().contains("_ai_sessions_fallback_"), + "nothing is counted in the instance store for a workspace with storage: {usage}" + ); + + // Pointed at the instance store's own bucket, a storage of its own keeps its live + // backups there under the current generation, which no storage change deletes. + configure_primary_lfs_via_route(&base, &instance_root).await?; + assert_eq!(push_whole().await?.status(), 200); + assert!(!files_under(&in_instance).is_empty()); + let same = list(&base, "SECRET_TOKEN").await?; + configure_primary_lfs_via_route(&base, &instance_root).await?; + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + assert!(!files_under(&in_instance).is_empty()); + let listing = list(&base, "SECRET_TOKEN").await?; + assert!(listing.get("fallback").is_none(), "{listing}"); + assert_eq!(listing["backup_generation"], same["backup_generation"]); + assert_eq!(listing["sessions"][0]["id"], "s1"); + + // Back to no storage of its own, the fallback answers; a plan switched to Pro while the + // instance store stays loaded stops it at once, for the listing and the push alike. + let resp = authed( + client().post(format!("{base}/workspaces/edit_large_file_storage_config")), + "SECRET_TOKEN", + ) + .json(&json!({ "large_file_storage": null })) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + assert_eq!(list(&base, "SECRET_TOKEN").await?["fallback"], true); + let _enterprise_again = ResetLicensePlan; + windmill_common::ee::LICENSE_KEY_ID.store(std::sync::Arc::new("test_pro".to_string())); + assert_eq!(list(&base, "SECRET_TOKEN").await?["enabled"], false); + let resp = push_whole().await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let pushed: Value = resp.json().await?; + assert_eq!(pushed["enabled"], false, "{pushed}"); + + Ok(()) +} diff --git a/backend/windmill-api-workspaces/src/ai_session_backups.rs b/backend/windmill-api-workspaces/src/ai_session_backups.rs index bbdbfbc8c9..46c91d5354 100644 --- a/backend/windmill-api-workspaces/src/ai_session_backups.rs +++ b/backend/windmill-api-workspaces/src/ai_session_backups.rs @@ -1,5 +1,6 @@ -//! What the workspace key rotation and the AI session backup routes -//! (`windmill-api/src/ai_sessions.rs`) share about the backups in the workspace storage. +//! What the workspace key rotation, the workspace storage settings and the AI session backup +//! routes (`windmill-api/src/ai_sessions.rs`) share about the backups: the store they live +//! in, and what a rotation or a storage change deletes. //! //! The backups are ciphertext under the workspace key and live under a prefix named by a //! generation the rotation bumps (`workspace_settings.ai_sessions_backup_generation`) in the @@ -13,17 +14,29 @@ //! one to read with and nothing to rewrite in place. A generation is never reused, so no //! deletion, however late, can touch live objects; a rotation that fails before its commit //! bumps nothing and deletes nothing; two rotations racing serialize on the key row. +//! +//! A workspace without storage of its own keeps its backups in the instance object store +//! instead, under the same layout and key, while `ai_sessions_instance_storage_fallback` +//! allows it. Configuring a storage for such a workspace bumps the generation in the +//! transaction that sets it, so everything the workspace left in any instance store sits +//! under a generation the routes never read again: a later return to the instance store, +//! whichever it is by then, starts from a newer one. That is what lets a storage change +//! delete the older generations from the instance store without fencing against what +//! happens next, and a browser retire a removal owed to an instance store once the +//! workspace's own storage answered. use std::sync::Arc; -use futures::TryStreamExt; +use futures::{StreamExt, TryStreamExt}; use windmill_common::error::{Error, Result}; use windmill_common::utils::calculate_hash; use windmill_common::DB; use windmill_object_store::object_store_reexports::{ ObjectStore, ObjectStoreError, Path as ObjectPath, }; -use windmill_object_store::{object_store_error_to_error, ObjectStoreResource}; +use windmill_object_store::{ + object_store_error_to_error, object_store_location, ObjectStoreResource, +}; use windmill_types::s3::LargeFileStorage; /// The root of every AI session backup key in a workspace's storage. @@ -31,6 +44,9 @@ pub const ROOT: &str = "windmill_ai_sessions"; /// The push body cap: no object written through the routes is larger. One that is was /// planted by whoever holds the bucket's credentials, and is left unread. pub const MAX_OBJECT_BYTES: usize = 32 * 1024 * 1024; +/// The storage name the workspace's backups in the instance store count under in its +/// storage usage, next to `_default_` and the secondary storages. +pub const FALLBACK_STORAGE: &str = "_ai_sessions_fallback_"; const IO_CONCURRENCY: usize = 8; @@ -39,34 +55,76 @@ pub fn generation_prefix(w_id: &str, generation: i64) -> String { format!("{ROOT}/{w_id}/g{generation}") } +/// The prefix of everything the workspace ever backed up, whatever the generation. +fn workspace_prefix(w_id: &str) -> ObjectPath { + ObjectPath::from(format!("{ROOT}/{w_id}")) +} + /// Names the storage the backups are in, by what locates its objects (endpoint, region, /// bucket; never the credentials, which rotate), so a browser tells that its sync state was /// recorded against another storage; the generation, answered alongside, tells it a /// rotation happened in this one. pub fn storage_id(resource: &ObjectStoreResource) -> String { - let location = match resource { - ObjectStoreResource::S3(s) => format!( - "s3:{}:{}:{}:{}", - s.endpoint, - s.port.unwrap_or_default(), - s.region, - s.bucket - ), - ObjectStoreResource::Azure(a) => format!( - "azure:{}:{}:{}", - a.endpoint.as_deref().unwrap_or_default(), - a.account_name, - a.container_name - ), - ObjectStoreResource::Gcs(g) => format!("gcs:{}", g.bucket), - ObjectStoreResource::Filesystem(f) => format!("fs:{}", f.root_path), - }; - calculate_hash(&location)[..16].to_string() + calculate_hash(&object_store_location(resource))[..16].to_string() } -/// The workspace's primary storage, resolved without a caller: a rotation runs the -/// deletion off its own request. -async fn primary_store(db: &DB, w_id: &str) -> Result>> { +/// Where a workspace's backups live: its primary storage, or the instance object store +/// standing in for it. +pub struct BackupStore { + pub store: Arc, + pub storage_id: String, + pub fallback: bool, +} + +/// The instance object store, for a workspace without storage of its own: loaded from +/// settings that say where its objects are, and not turned off by +/// `ai_sessions_instance_storage_fallback`, which is on unless set to false. Named like a +/// workspace storage, by that location, in a namespace of its own. Never on the Pro plan, +/// checked on every call: a store loaded before a switch to Pro stays loaded. Never in a +/// build without `private`, which has neither workspace storage nor the quota the fallback +/// counts toward. +/// +/// Authorizes nothing, and the store reaches every workspace's objects: the caller must have +/// authorized the user for the workspace and keep what it reads and writes under that +/// user's prefix in it, as the backup routes do. +pub async fn fallback_store(db: &DB) -> Result> { + #[cfg(not(feature = "private"))] + { + let _ = db; + Ok(None) + } + #[cfg(feature = "private")] + { + if matches!( + windmill_common::ee_oss::get_license_plan().await, + windmill_common::ee_oss::LicensePlan::Pro + ) { + return Ok(None); + } + let Some((store, Some(location))) = + windmill_object_store::get_object_store_with_location().await + else { + return Ok(None); + }; + let setting = windmill_common::global_settings::load_value_from_global_settings( + db, + windmill_common::global_settings::AI_SESSIONS_INSTANCE_STORAGE_FALLBACK_SETTING, + ) + .await?; + if matches!(setting, Some(serde_json::Value::Bool(false))) { + return Ok(None); + } + Ok(Some(BackupStore { + storage_id: calculate_hash(&format!("instance:{location}"))[..16].to_string(), + store, + fallback: true, + })) + } +} + +/// The workspace's primary storage, resolved without a caller: a rotation runs its deletion +/// off its own request. +async fn primary_store(db: &DB, w_id: &str) -> Result> { let Some(lfs_json) = sqlx::query_scalar!( "SELECT large_file_storage FROM workspace_settings WHERE workspace_id = $1", w_id @@ -91,9 +149,19 @@ async fn primary_store(db: &DB, w_id: &str) -> Result Result> { + if let Some(primary) = primary_store(db, w_id).await? { + return Ok(Some(primary)); + } + fallback_store(db).await } /// The generation an object key sits under, `None` for a key of no generation (an older @@ -107,40 +175,41 @@ fn generation_of(w_id: &str, key: &ObjectPath) -> Option { .ok() } -/// Deletes, off the request and as the listing streams, every object of the workspace's -/// backups from a generation older than `current`, once the rotation that made `current` -/// the generation has committed: nothing writes there any more but a push that resolved its -/// prefix before the commit, junk the browser's next push of that session rewrites under the -/// current prefix, as is anything a deletion cut short left behind. For the rotation route, -/// which authorized its caller as a superadmin. +/// Deletes, as the listing streams, every object of the workspace's backups in the store +/// from a generation older than `current`. +async fn delete_older(store: &Arc, w_id: &str, current: i64) -> Result<()> { + store + .list(Some(&workspace_prefix(w_id))) + .map_err(object_store_error_to_error) + .try_for_each_concurrent(IO_CONCURRENCY, |meta| async move { + if generation_of(w_id, &meta.location).is_some_and(|g| g >= current) { + return Ok(()); + } + match store.delete(&meta.location).await { + Ok(()) | Err(ObjectStoreError::NotFound { .. }) => Ok(()), + Err(e) => Err(object_store_error_to_error(e)), + } + }) + .await +} + +/// Deletes, off the request, every object of the workspace's backups from a generation +/// older than `current`, once the rotation that made `current` the generation has +/// committed: nothing writes there any more but a push that resolved its prefix before the +/// commit, junk the browser's next push of that session rewrites under the current prefix, +/// as is anything a deletion cut short left behind. For the rotation route, which +/// authorized its caller as a superadmin. pub(crate) fn spawn_delete_older(db: DB, w_id: String, current: i64) { tokio::spawn(async move { - let store = match primary_store(&db, &w_id).await { - Ok(Some(store)) => store, + let store = match workspace_store(&db, &w_id).await { + Ok(Some(store)) => store.store, Ok(None) => return, Err(e) => { tracing::warn!("older AI session backups of {w_id} left in place: {e:#}"); return; } }; - let prefix = ObjectPath::from(format!("{ROOT}/{w_id}")); - let deleted = store - .list(Some(&prefix)) - .map_err(object_store_error_to_error) - .try_for_each_concurrent(IO_CONCURRENCY, |meta| { - let (store, w_id) = (&store, &w_id); - async move { - if generation_of(w_id, &meta.location).is_some_and(|g| g >= current) { - return Ok(()); - } - match store.delete(&meta.location).await { - Ok(()) | Err(ObjectStoreError::NotFound { .. }) => Ok(()), - Err(e) => Err(object_store_error_to_error(e)), - } - } - }) - .await; - match deleted { + match delete_older(&store, &w_id, current).await { Ok(()) => { tracing::info!("deleted the AI session backups of {w_id} older than g{current}") } @@ -148,3 +217,57 @@ pub(crate) fn spawn_delete_older(db: DB, w_id: String, current: i64) { } }); } + +/// Deletes, off the request, what the workspace's backups left in the instance store under +/// a generation older than `current`, the one a storage settings change committed. Nothing +/// reads there: the routes use the workspace's own storage, or, back in the instance store, +/// `current` or a newer generation, since configuring a storage over the fallback bumped +/// it. So it runs whatever the storage is now and whatever the setting says (copies from +/// when it was on may be there), and a deletion that is slow, cut short or overtaken by a +/// later change deletes nothing live. For the storage settings route, which authorized its +/// caller as a workspace admin. +pub(crate) fn spawn_delete_fallback(w_id: String, current: i64) { + tokio::spawn(async move { + let Some(instance) = windmill_object_store::get_object_store().await else { + return; + }; + match delete_older(&instance, &w_id, current).await { + Ok(()) => tracing::info!( + "deleted the AI session backups of {w_id} older than g{current} from the instance store" + ), + Err(e) => tracing::warn!( + "deleting the AI session backups of {w_id} from the instance store: {e:#}" + ), + } + }); +} + +/// The bytes of the workspace's backups in the instance store, for its storage usage while +/// it has no storage of its own (once it has one nothing writes there, and the change +/// deleted what was): `None` when it has one, when there is no instance store, or when +/// there is nothing, so no empty usage entry shows up. Whether the setting is on or off, +/// since copies from when it was on may be there. +/// +/// Authorizes nothing: for the storage usage recount, which reports a total for the +/// workspace it was run for and hands out nothing it read. +pub async fn fallback_bytes(db: &DB, w_id: &str) -> Result> { + let has_storage = sqlx::query_scalar!( + r#"SELECT large_file_storage IS NOT NULL AS "has_storage!" FROM workspace_settings WHERE workspace_id = $1"#, + w_id + ) + .fetch_optional(db) + .await? + .unwrap_or(false); + if has_storage { + return Ok(None); + } + let Some(instance) = windmill_object_store::get_object_store().await else { + return Ok(None); + }; + let mut total: i64 = 0; + let mut stream = instance.list(Some(&workspace_prefix(w_id))); + while let Some(meta) = stream.next().await { + total += meta.map_err(object_store_error_to_error)?.size as i64; + } + Ok((total > 0).then_some(total)) +} diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 1a3c24f096..c240948105 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -2093,6 +2093,17 @@ async fn edit_large_file_storage_config( serde_json::to_value::(lfs_config) .map_err(|err| Error::internal_err(err.to_string()))?; + // A workspace whose AI session backups fell back to the instance store leaves it + // here: the generation moves on, so nothing it left in any instance store is read + // again, whichever one a later return to the fallback finds (`ai_session_backups`). + sqlx::query!( + "UPDATE workspace_settings SET ai_sessions_backup_generation = \ + ai_sessions_backup_generation + 1 \ + WHERE workspace_id = $1 AND large_file_storage IS NULL", + &w_id + ) + .execute(&mut *tx) + .await?; sqlx::query!( "UPDATE workspace_settings SET large_file_storage = $1 WHERE workspace_id = $2", serialized_lfs_config, @@ -2108,8 +2119,23 @@ async fn edit_large_file_storage_config( .execute(&mut *tx) .await?; } + let backups_generation = sqlx::query_scalar!( + "SELECT ai_sessions_backup_generation FROM workspace_settings WHERE workspace_id = $1", + &w_id + ) + .fetch_optional(&mut *tx) + .await?; tx.commit().await?; + // Read by nothing any more, whatever the storage is now: what the AI session backups + // left in the instance store under a generation older than the one just committed. + #[cfg(feature = "parquet")] + if let Some(generation) = backups_generation { + crate::ai_session_backups::spawn_delete_fallback(w_id.clone(), generation); + } + #[cfg(not(feature = "parquet"))] + let _ = backups_generation; + // Trigger git sync for large file storage changes handle_deployment_metadata( &authed.email, diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 9f742a129b..63e8dd46ea 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -13222,6 +13222,9 @@ paths: backup_generation: type: integer description: bumped by every workspace key rotation; sync state recorded under another one is void + fallback: + type: boolean + description: the storage answered from is the instance object store, standing in for a workspace without storage of its own; a removal owed to it is retired by any answer from the workspace's own storage sessions: type: array description: the newest 500 at most @@ -13273,6 +13276,8 @@ paths: type: string backup_generation: type: integer + fallback: + type: boolean sessions: type: array items: @@ -13327,6 +13332,8 @@ paths: type: string backup_generation: type: integer + fallback: + type: boolean results: type: array items: diff --git a/backend/windmill-api/src/ai_sessions.rs b/backend/windmill-api/src/ai_sessions.rs index fca9c35fca..776b3a0561 100644 --- a/backend/windmill-api/src/ai_sessions.rs +++ b/backend/windmill-api/src/ai_sessions.rs @@ -30,7 +30,7 @@ use serde_json::value::RawValue; use std::sync::Arc; use windmill_api_auth::is_effectively_unscoped; use windmill_api_workspaces::ai_session_backups::{ - generation_prefix, storage_id, MAX_OBJECT_BYTES, + fallback_store, generation_prefix, storage_id, MAX_OBJECT_BYTES, }; use windmill_common::error::{Error, JsonResult, Result}; use windmill_common::utils::calculate_hash; @@ -101,6 +101,9 @@ struct Backend { /// owed to the storage alone (a rotation deleted the older generation's copy anyway). storage_id: String, generation: i64, + /// The store is the instance object store standing in for a workspace without storage + /// of its own (`ai_session_backups::fallback_store`). + fallback: bool, } impl Backend { @@ -408,34 +411,56 @@ fn require_json_object(kind: &str, raw: &RawValue, max_bytes: usize) -> Result<( Ok(()) } -/// `None` when the workspace has nowhere to keep backups: no primary storage configured, or -/// the admin switched them off. Both read as `enabled: false` so the browser stops trying. +/// `None` when the workspace has nowhere to keep backups: no primary storage configured and +/// no instance store to stand in, or the admin switched them off. Both read as +/// `enabled: false` so the browser stops trying. async fn backend(authed: &ApiAuthed, db: &DB, w_id: &str) -> Result> { - let (disabled, generation) = sqlx::query_as::<_, (Option, i64)>( - "SELECT (ai_config->>'sessions_storage_disabled')::bool, ai_sessions_backup_generation \ - FROM workspace_settings WHERE workspace_id = $1", + let (disabled, generation, has_storage) = sqlx::query_as::<_, (Option, i64, bool)>( + "SELECT (ai_config->>'sessions_storage_disabled')::bool, ai_sessions_backup_generation, \ + large_file_storage IS NOT NULL FROM workspace_settings WHERE workspace_id = $1", ) .bind(w_id) .fetch_optional(db) .await? - .unwrap_or((None, 0)); + .unwrap_or((None, 0, false)); if disabled.unwrap_or(false) { return Ok(None); } - let (_, resource) = - crate::job_helpers_oss::get_workspace_s3_resource(authed, db, None, w_id, None).await?; - let Some(resource) = resource else { - return Ok(None); + // Decided from the row the generation came from: the instance store is written only + // under a generation read while the workspace had no storage of its own, which + // configuring one moves past (`ai_session_backups`). + let (store, storage_id, fallback) = if has_storage { + let (_, resource) = + crate::job_helpers_oss::get_workspace_s3_resource(authed, db, None, w_id, None).await?; + let Some(resource) = resource else { + return Ok(None); + }; + ( + build_object_store_client(&resource).await?, + storage_id(&resource), + false, + ) + } else { + // The instance store stands in, under the same layout and the same key. + match fallback_store(db).await? { + Some(f) => (f.store, f.storage_id, true), + None => return Ok(None), + } }; - let store = build_object_store_client(&resource).await?; let user = calculate_hash(&authed.email); // Keyed per user, not per workspace: anyone who can write the bucket could otherwise copy // another member's ciphertext under their own prefix and have `pull` decrypt it for them. let key = get_workspace_key(w_id, db).await?; let mc = crypt_from_key_with_suffix(&key, &user); - let storage_id = storage_id(&resource); let prefix = format!("{}/{user}", generation_prefix(w_id, generation)); - Ok(Some(Backend { store, mc, prefix, storage_id, generation })) + Ok(Some(Backend { + store, + mc, + prefix, + storage_id, + generation, + fallback, + })) } #[derive(Serialize)] @@ -455,6 +480,12 @@ struct ListResponse { storage_id: Option, #[serde(skip_serializing_if = "Option::is_none")] backup_generation: Option, + /// The storage is the instance store standing in for a workspace without one of its + /// own; a removal owed to it is retired by any answer from the workspace's own storage + /// once it has one (configuring it moved the generation past everything the workspace + /// left in any instance store). + #[serde(skip_serializing_if = "std::ops::Not::not")] + fallback: bool, sessions: Vec, /// The user has more sessions than the answer names. #[serde(skip_serializing_if = "std::ops::Not::not")] @@ -474,6 +505,7 @@ async fn list( enabled: false, storage_id: None, backup_generation: None, + fallback: false, sessions: vec![], truncated: false, })); @@ -526,6 +558,7 @@ async fn list( enabled: true, storage_id: Some(backend.storage_id.clone()), backup_generation: Some(backend.generation), + fallback: backend.fallback, sessions, truncated, })) @@ -594,6 +627,8 @@ struct PullResponse { storage_id: Option, #[serde(skip_serializing_if = "Option::is_none")] backup_generation: Option, + #[serde(skip_serializing_if = "std::ops::Not::not")] + fallback: bool, sessions: Vec, /// Ids that did not fit the response budget; ask for them again. deferred: Vec, @@ -839,6 +874,7 @@ async fn pull( enabled: false, storage_id: None, backup_generation: None, + fallback: false, sessions: vec![], deferred: vec![], })); @@ -861,6 +897,7 @@ async fn pull( enabled: true, storage_id: Some(backend.storage_id), backup_generation: Some(backend.generation), + fallback: backend.fallback, sessions, deferred, })) @@ -952,6 +989,8 @@ struct PushResponse { storage_id: Option, #[serde(skip_serializing_if = "Option::is_none")] backup_generation: Option, + #[serde(skip_serializing_if = "std::ops::Not::not")] + fallback: bool, results: Vec, } @@ -1241,6 +1280,7 @@ async fn push( enabled: false, storage_id: None, backup_generation: None, + fallback: false, results: vec![], })); }; @@ -1294,16 +1334,16 @@ async fn push( results.push(PushResult { id: sid.clone(), error, needs_whole: false }); } // Overwrites and deletes make this an over-count; the periodic recount the quota check - // schedules once usage is stale settles it. + // schedules once usage is stale settles it. Bytes in the instance store count under a + // name of their own, which the recount lists there. #[cfg(not(feature = "enterprise"))] if written > 0 { - crate::job_helpers_oss::bump_storage_usage( - &db, - &w_id, - windmill_object_store::DEFAULT_STORAGE, - written as i64, - ) - .await; + let storage = if backend.fallback { + windmill_api_workspaces::ai_session_backups::FALLBACK_STORAGE + } else { + windmill_object_store::DEFAULT_STORAGE + }; + crate::job_helpers_oss::bump_storage_usage(&db, &w_id, storage, written as i64).await; } #[cfg(feature = "enterprise")] let _ = written; @@ -1311,6 +1351,7 @@ async fn push( enabled: true, storage_id: Some(backend.storage_id), backup_generation: Some(backend.generation), + fallback: backend.fallback, results, })) } diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index 49efa26f60..ef63fc2347 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -84,6 +84,11 @@ pub const SANDBOX_REGISTRY_AUTH_SETTING: &str = "sandbox_registry_auth"; // windmill-worker/src/ssh_executor_ee.rs. pub const SSH_EXECUTION_SETTING: &str = "ssh_execution_enabled"; pub const OBJECT_STORE_CONFIG_SETTING: &str = "object_store_cache_config"; +/// Whether the instance object store stands in for a workspace without storage of its own +/// as the place its members' AI sessions are backed up to. On unless the row says `false`; +/// inert without an instance object store. +pub const AI_SESSIONS_INSTANCE_STORAGE_FALLBACK_SETTING: &str = + "ai_sessions_instance_storage_fallback"; /// Compile a newly deployed script's binary right after its dependency job and push it /// to the instance object store, so the first run does not pay the compile. Inert unless /// instance object storage is configured — without it the binary would only ever land in diff --git a/backend/windmill-common/src/instance_config.rs b/backend/windmill-common/src/instance_config.rs index 1dd3396809..de3fd7684a 100644 --- a/backend/windmill-common/src/instance_config.rs +++ b/backend/windmill-common/src/instance_config.rs @@ -264,6 +264,8 @@ pub struct GlobalSettings { pub disable_hub: Option, #[serde(skip_serializing_if = "Option::is_none")] pub auto_build_binary_on_deploy: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub ai_sessions_instance_storage_fallback: Option, // String settings #[serde(skip_serializing_if = "Option::is_none")] diff --git a/backend/windmill-object-store/src/lib.rs b/backend/windmill-object-store/src/lib.rs index 25dc6ea19c..edc3ea9149 100644 --- a/backend/windmill-object-store/src/lib.rs +++ b/backend/windmill-object-store/src/lib.rs @@ -118,6 +118,10 @@ pub fn object_store_error_to_error(err: object_store::Error) -> error::Error { pub struct ExpirableObjectStore { pub store: Arc, pub refresh: Option, + /// What locates the store's objects ([`object_store_location`]), for a store built from + /// settings. Kept with the store rather than read off the settings again, so a server + /// whose reload is still pending never names one store by another's location. + pub location: Option, } #[cfg(feature = "parquet")] @@ -155,7 +159,7 @@ impl ObjectStoreRefresh { #[cfg(feature = "parquet")] impl From> for ExpirableObjectStore { fn from(store: Arc) -> Self { - Self { store, refresh: None } + Self { store, refresh: None, location: None } } } @@ -197,6 +201,15 @@ static CACHE_OVERRIDE_GENERATION: std::sync::atomic::AtomicU64 = async fn resolve_object_store( settings_lock: &RwLock>, ) -> Option> { + resolve_object_store_with_location(settings_lock) + .await + .map(|(store, _)| store) +} + +#[cfg(feature = "parquet")] +async fn resolve_object_store_with_location( + settings_lock: &RwLock>, +) -> Option<(Arc, Option)> { let settings = settings_lock.read().await; let Some(s) = settings.as_ref() else { return None; @@ -212,18 +225,18 @@ async fn resolve_object_store( // A reload may have installed a different store while the credentials were // being minted; that one reflects newer config, so the refresh is stale. Some(current) if !Arc::ptr_eq(¤t.store, &refreshed_from) => { - Some(current.store.clone()) + Some((current.store.clone(), current.location.clone())) } Some(_) => { - let arc = new_store.store.clone(); + let found = (new_store.store.clone(), new_store.location.clone()); *settings = Some(new_store); - Some(arc) + Some(found) } // Cleared while refreshing. None => None, } } - _ => Some(s.store.clone()), + _ => Some((s.store.clone(), s.location.clone())), } } @@ -232,6 +245,13 @@ pub async fn get_object_store() -> Option> { resolve_object_store(&OBJECT_STORE_SETTINGS).await } +/// The instance object store with what locates its objects ([`object_store_location`]), +/// read together; the location is `None` for a store installed without settings. +#[cfg(feature = "parquet")] +pub async fn get_object_store_with_location() -> Option<(Arc, Option)> { + resolve_object_store_with_location(&OBJECT_STORE_SETTINGS).await +} + /// The store the dependency cache reads and writes: the worker group's override when it has one, /// the instance object store otherwise. Anything the server must also reach goes through /// [`get_object_store`] instead. @@ -422,20 +442,22 @@ pub async fn reload_object_store_setting(db: &windmill_common::DB) -> ObjectStor tracing::error!("S3 cache is not available for pro plan"); return ObjectStoreReload::Never; } - *s3_cache_settings = build_s3_client_from_settings(S3Settings { - bucket: None, - region: None, - access_key: None, - secret_key: None, - endpoint: None, - store_logs: None, - path_style: None, - allow_http: None, - port: None, - }) + *s3_cache_settings = build_object_store_from_settings( + ObjectSettings::S3(S3Settings { + bucket: None, + region: None, + access_key: None, + secret_key: None, + endpoint: None, + store_logs: None, + path_style: None, + allow_http: None, + port: None, + }), + Some(db), + ) .await .ok() - .map(|x| ExpirableObjectStore::from(x)) } else { *s3_cache_settings = None; } @@ -887,19 +909,49 @@ impl ObjectStore for FilesystemStoreIgnoringAttributes { } } +/// What locates a store's objects: endpoint, port, region and bucket (or account and +/// container, or root), never the credentials, which rotate. Two stores with the same +/// location hold the same objects. +pub fn object_store_location(resource: &ObjectStoreResource) -> String { + match resource { + ObjectStoreResource::S3(s) => format!( + "s3:{}:{}:{}:{}", + s.endpoint, + s.port.unwrap_or_default(), + s.region, + s.bucket + ), + ObjectStoreResource::Azure(a) => format!( + "azure:{}:{}:{}", + a.endpoint.as_deref().unwrap_or_default(), + a.account_name, + a.container_name + ), + ObjectStoreResource::Gcs(g) => format!("gcs:{}", g.bucket), + ObjectStoreResource::Filesystem(f) => format!("fs:{}", f.root_path), + } +} + #[cfg(feature = "parquet")] pub async fn build_object_store_from_settings( settings: ObjectSettings, init_private_key: Option<&windmill_common::DB>, ) -> error::Result { + let located = + |store: Arc, resource: ObjectStoreResource| ExpirableObjectStore { + store, + refresh: None, + location: Some(object_store_location(&resource)), + }; match settings { - ObjectSettings::S3(s3_settings) => build_s3_client_from_settings(s3_settings) - .await - .map(|x| ExpirableObjectStore::from(x)), - ObjectSettings::Azure(azure_settings) => { - let azure_blob_resource = azure_settings; - build_azure_blob_client(&azure_blob_resource).map(|x| ExpirableObjectStore::from(x)) + ObjectSettings::S3(s3_settings) => { + let s3_resource = s3_resource_from_settings(s3_settings); + build_s3_client(&s3_resource) + .await + .map(|x| located(x, ObjectStoreResource::S3(s3_resource))) } + ObjectSettings::Azure(azure_settings) => build_azure_blob_client(&azure_settings) + .map(|x| located(x, ObjectStoreResource::Azure(azure_settings))), ObjectSettings::AwsOidc(ref s3_aws_oidc_settings) => { let token_generator = crate::job_s3_helpers_oss::TokenGenerator::AsServerInstance(); let res = crate::job_s3_helpers_oss::generate_s3_aws_oidc_resource( @@ -914,17 +966,14 @@ pub async fn build_object_store_from_settings( .map(|x| ExpirableObjectStore { store: x, refresh: Some(ObjectStoreRefresh::new(settings.clone(), res.expiration())), + location: Some(object_store_location(&res)), }) } - ObjectSettings::Gcs(gcs_settings) => { - let gcs_resource = gcs_settings; - build_gcs_client(&gcs_resource) - .await - .map(|x| ExpirableObjectStore::from(x)) - } - ObjectSettings::Filesystem(fs) => { - build_filesystem_client(&fs.root_path).map(|x| ExpirableObjectStore::from(x)) - } + ObjectSettings::Gcs(gcs_settings) => build_gcs_client(&gcs_settings) + .await + .map(|x| located(x, ObjectStoreResource::Gcs(gcs_settings))), + ObjectSettings::Filesystem(fs) => build_filesystem_client(&fs.root_path) + .map(|x| located(x, ObjectStoreResource::Filesystem(fs))), } } @@ -937,14 +986,14 @@ fn none_if_empty(s: Option) -> Option { } } +/// The S3 resource instance settings resolve to, the environment filling in what they +/// leave out. #[cfg(feature = "parquet")] -pub async fn build_s3_client_from_settings( - settings: S3Settings, -) -> error::Result> { +fn s3_resource_from_settings(settings: S3Settings) -> S3Resource { let region = none_if_empty(settings.region) .unwrap_or_else(|| std::env::var("AWS_REGION").unwrap_or_else(|_| "us-east-1".to_string())); - let s3_resource = S3Resource { + S3Resource { endpoint: none_if_empty(settings.endpoint).unwrap_or_else(|| { std::env::var("S3_ENDPOINT").unwrap_or_else(|_| format!("s3.{region}.amazonaws.com")) }), @@ -959,9 +1008,7 @@ pub async fn build_s3_client_from_settings( port: settings.port, token: None, expiration: None, - }; - - build_s3_client(&s3_resource).await + } } // Resolving the default chain goes over the network (ECS/IMDS) on instances relying on an @@ -2624,6 +2671,36 @@ mod tests { reload_cache_object_store_override(&db, None).await; } + /// A store built from settings is located by where its objects are, not by how the + /// client describes itself: an S3 client prints only its bucket, so the same bucket name + /// on another endpoint would otherwise pass for the same store. + #[cfg(feature = "parquet")] + #[tokio::test] + async fn test_settings_store_location_tells_endpoints_apart() { + let s3 = |endpoint: &str| { + ObjectSettings::S3(S3Settings { + bucket: Some("windmill".to_string()), + region: Some("us-east-1".to_string()), + access_key: Some("key".to_string()), + secret_key: Some("secret".to_string()), + endpoint: Some(endpoint.to_string()), + allow_http: Some(true), + path_style: Some(true), + store_logs: None, + port: None, + }) + }; + let a = build_object_store_from_settings(s3("minio.internal:9000"), None) + .await + .unwrap(); + let b = build_object_store_from_settings(s3("s3.us-east-1.amazonaws.com"), None) + .await + .unwrap(); + assert_eq!(a.store.to_string(), b.store.to_string()); + assert!(a.location.is_some()); + assert_ne!(a.location, b.location); + } + // --- get_logs_from_store test --- #[cfg(feature = "parquet")] diff --git a/docs/ai-session-backups.md b/docs/ai-session-backups.md index 0b4c550456..3e0c747b8a 100644 --- a/docs/ai-session-backups.md +++ b/docs/ai-session-backups.md @@ -129,11 +129,58 @@ against it for the same reason. The feature is on wherever the workspace has primary storage, and off with `ai_config.sessions_storage_disabled` (the `copilot_disabled` pattern: no migration, carried by settings export and the CLI). A build without `parquet` has no routes (404), a workspace without -storage answers `enabled: false`; either turns the backup off for ten minutes, after which the +storage and nothing to stand in for it answers `enabled: false`; either turns the backup off +for ten minutes, after which the page asks again on its own (a flush for whatever is pending, and a restore), and the AI settings page tells the mirror at once when the switch is saved there (the off state is forgotten, the rows that went stale are marked again, a restore runs). +## The instance store standing in + +A workspace without storage of its own keeps its backups in the instance object store +(`object_store_cache_config`, loaded the way every other use of it is, so never with +`DISABLE_S3_STORE`; the plan is checked on every request and Pro never falls back, since a +store loaded before a switch to Pro stays loaded), under the same layout and the same +per-user key, +while the instance setting `ai_sessions_instance_storage_fallback` allows it (on unless set +to false; the instance settings page shows it under Object Storage). A build without +`private` has neither workspace storage nor the quota below, and never falls back. Every +answer says which kind of store it came from (`fallback`), and the instance store is named +(`storage_id`) by what locates its objects, the endpoint, region and bucket its settings +resolve to, in a namespace of its own: moving the instance store to another endpoint under +the same bucket name is a storage switch for the browsers, and a workspace bucket is never +taken for it. The location is kept with the loaded store, so a server whose reload is still +pending names the store it writes to. A route decides between the workspace's storage and +the instance store from the row it reads the generation from, so a push lands in the +instance store only under a generation read while the workspace had no storage. The store a +workspace's backups live in is resolved in one place +(`ai_session_backups::workspace_store`), for the routes and for the rotation's deletion of +older generations alike. + +Configuring a storage for a workspace that had none (`edit_large_file_storage_config`) bumps +the backup generation in the transaction that sets it, so everything the workspace left in +any instance store sits under a generation the routes never read again: a later return to +the instance store, whichever it is by then, starts from a newer one. Every storage settings +change then deletes from the instance store, off the request, the workspace's generations +older than the one it committed, whether the setting is on or off (copies from when it was +on may be there). Nothing live is older, whatever happens next: a deletion that is slow, cut +short, or overtaken by the storage being dropped or pointed at the instance store's own +bucket touches only generations nothing reads. Dropping the storage bumps nothing and is a +switch like any other: the rows go stale, the sessions are pushed whole into the instance +store, and the old bucket keeps its copy. On the browser side a removal owed to an instance +store (the row names it apart, `storageName` in `sessionMirrorPlan.ts`) is retired by any +answer from the workspace's own storage, since that storage being there means the +generation moved past the copy; one owed to a workspace storage still waits for that +storage, whatever the instance store answered. Copies a deletion missed stay in the +operator's bucket unread, as a deleted workspace's copies do. + +On CE the bytes in the instance store count toward the workspace's storage quota under a +storage name of their own (`_ai_sessions_fallback_`, listed by the periodic recount while +the workspace has no storage of its own, and left out when there are none), so a member +cannot fill the operator's bucket past what the workspace may use; on EE, where workspace +storage has no quota either, nothing bounds them but the per-push caps and the instance +setting. + ## Conflicts and deletion Last write wins across devices. The head carries no manifest; `pull` lists the session's prefix @@ -172,7 +219,8 @@ session whose pieces could not be written: recording it would let the next flush half-empty local state over the backup. Every answer names the storage it came from (`storage_id`, a hash of what locates the objects, -endpoint, region and bucket, not the credentials, which rotate) and the backup generation a +endpoint, region and bucket, not the credentials, which rotate; the instance store standing +in for a workspace without one is named apart, see above) and the backup generation a key rotation bumps (`backup_generation`). A sync row records both, and a row naming another storage or generation goes stale and its session is marked again: a workspace pointed at a new bucket, or whose key was rotated, holds nothing, and the server looks nowhere else, so diff --git a/frontend/src/lib/components/InstanceSettings.svelte b/frontend/src/lib/components/InstanceSettings.svelte index f51e459f77..f38c0d3892 100644 --- a/frontend/src/lib/components/InstanceSettings.svelte +++ b/frontend/src/lib/components/InstanceSettings.svelte @@ -134,11 +134,15 @@ } applyFormDefaults(nvalues) - // Apply select/select_python defaults so initialValues matches what InstanceSetting's $effect does + // Apply declared defaults before snapshotting initialValues, so a default shows without + // marking the form dirty: a select's mirrors InstanceSetting's $effect, a boolean's is + // what its toggle shows while the key is unset. for (const category of settingsKeys) { for (const s of settings[category]) { if ( - (s.fieldType === 'select' || s.fieldType === 'select_python') && + (s.fieldType === 'select' || + s.fieldType === 'select_python' || + s.fieldType === 'boolean') && nvalues[s.key] == undefined && s.defaultValue ) { diff --git a/frontend/src/lib/components/instanceSettings.ts b/frontend/src/lib/components/instanceSettings.ts index 0677648978..34079eae51 100644 --- a/frontend/src/lib/components/instanceSettings.ts +++ b/frontend/src/lib/components/instanceSettings.ts @@ -563,6 +563,17 @@ export const settings: Record = { storage: 'setting', ee_only: '' }, + { + label: 'Back AI sessions up to the instance object storage', + description: + "Browsers back their AI sessions up to their workspace's object storage, encrypted with the workspace key. When this is on and instance object storage is configured, a workspace without object storage of its own uses the instance object storage instead, under the same encryption; configuring a storage for the workspace moves its backups there and deletes what it kept in the instance storage. On by default; turn off to keep the AI sessions of such workspaces in the browser only.", + key: 'ai_sessions_instance_storage_fallback', + fieldType: 'boolean', + defaultValue: () => true, + storage: 'setting', + ee_only: '', + hideInQuickSetup: true + }, { label: 'Store audit logs in object storage', description: diff --git a/frontend/src/lib/components/sessions/sessionMirror.svelte.ts b/frontend/src/lib/components/sessions/sessionMirror.svelte.ts index b1ac4bcd06..a18e2febd6 100644 --- a/frontend/src/lib/components/sessions/sessionMirror.svelte.ts +++ b/frontend/src/lib/components/sessions/sessionMirror.svelte.ts @@ -55,10 +55,12 @@ import { import { artifactsFingerprint, headSig, + isFallbackStorage, jsonBytes, operationsOf, planSessionPush, splitEntry, + storageName, type ChatSnapshot, type MirrorSyncState, type PlannedPush, @@ -682,9 +684,10 @@ async function pushWorkspace( return 'transient' } if (!res.enabled) return 'off' - out.storageId = res.storage_id + const storageId = storageName(res.storage_id, res.fallback) + out.storageId = storageId out.generation = res.backup_generation - const answered = `${res.storage_id}:${res.backup_generation}` + const answered = `${storageId}:${res.backup_generation}` const errors = new Set() for (const r of res.results) { if (r.error) { @@ -705,7 +708,7 @@ async function pushWorkspace( // settle, the session goes again whole. if (a.answered !== undefined && a.answered !== answered) failed.add(entry.id) a.answered = answered - a.storageId = res.storage_id + a.storageId = storageId a.generation = res.backup_generation if (errors.has(entry.id)) failed.add(entry.id) } @@ -718,12 +721,20 @@ async function pushWorkspace( [mark.storageId, ...(mark.alsoIn ?? [])].filter((s): s is string => s !== undefined) ) // Answered from a storage holding no copy: the copies are still where they - // were, and the mark waits for those storages to answer. - if (holding.size === 0 || res.storage_id === undefined) out.removedDone.push(mark) - else if (holding.has(res.storage_id)) { - holding.delete(res.storage_id) + // were, and the mark waits for those storages to answer. The workspace's own + // storage answering retires every instance store's share too: configuring it + // moved the generation past all the workspace left in any instance store. + if (holding.size === 0 || storageId === undefined) out.removedDone.push(mark) + else { + const before = holding.size + holding.delete(storageId) + if (!res.fallback) { + for (const name of [...holding]) if (isFallbackStorage(name)) holding.delete(name) + } if (holding.size === 0) out.removedDone.push(mark) - else out.removedFrom.push({ id, key: mark.key, remaining: [...holding] }) + else if (holding.size < before) { + out.removedFrom.push({ id, key: mark.key, remaining: [...holding] }) + } } } } @@ -1204,7 +1215,7 @@ async function listWorkspace(ws: string, email: string): Promise { expect((await __syncRowsForTesting(EMAIL)).some((r) => r.id === 'sr3')).toBe(false) }) + it("retires a removal owed to any instance store once the workspace's own storage answered", async () => { + const s: Session = { id: 'fb1', name: 'session-1', createdAt: 1, workspace_id: 'ws' } + sessionState.sessions = [s] + await putSession(s) + pushMock.mockResolvedValueOnce({ + enabled: true, + storage_id: 'I1', + fallback: true, + results: [{ id: 'fb1' }] + }) + await __flushForTesting() + // The operator moved the instance store: the session goes whole to the new one, and + // the row remembers the copy the old one keeps. + await putSession({ ...s, summary: 'changed' }) + pushMock.mockResolvedValue({ + enabled: true, + storage_id: 'I2', + fallback: true, + results: [{ id: 'fb1' }] + }) + await __flushForTesting() + await __flushForTesting() + const row = (await __syncRowsForTesting(EMAIL)).find((r) => r.id === 'fb1') + expect(row?.storageId).toBe('instance:I2') + expect(row?.alsoIn).toEqual(['instance:I1']) + + // The workspace got a storage of its own meanwhile, which moved the generation past + // everything it left in either instance store: that storage's answer settles the + // removal. + deleteSession('fb1') + await flush() + pushMock.mockResolvedValue({ enabled: true, storage_id: 'A', results: [{ id: 'fb1' }] }) + await __flushForTesting() + expect(pushMock.mock.lastCall?.[0].requestBody.removed).toEqual(['fb1']) + expect(removalKeys()).toEqual([]) + expect((await __syncRowsForTesting(EMAIL)).some((r) => r.id === 'fb1')).toBe(false) + }) + + it('waits for the storage a workspace dropped, whatever the instance store answered', async () => { + const s: Session = { id: 'fb2', name: 'session-1', createdAt: 1, workspace_id: 'ws' } + sessionState.sessions = [s] + await putSession(s) + pushMock.mockResolvedValueOnce({ enabled: true, storage_id: 'A', results: [{ id: 'fb2' }] }) + await __flushForTesting() + // The workspace dropped its storage: the session goes whole to the instance store, + // and the row remembers the copy A keeps. + await putSession({ ...s, summary: 'changed' }) + pushMock.mockResolvedValue({ + enabled: true, + storage_id: 'I', + fallback: true, + results: [{ id: 'fb2' }] + }) + await __flushForTesting() + await __flushForTesting() + const row = (await __syncRowsForTesting(EMAIL)).find((r) => r.id === 'fb2') + expect(row?.storageId).toBe('instance:I') + expect(row?.alsoIn).toEqual(['A']) + + // Deleted while on the instance store: its copy goes, and the mark waits for A. + deleteSession('fb2') + await flush() + await __flushForTesting() + expect(removalKeys()).toEqual(['r::fb2::ws']) + expect((await __syncRowsForTesting(EMAIL)).find((r) => r.id === 'fb2')?.storageId).toBe('A') + pushMock.mockResolvedValue({ enabled: true, storage_id: 'A', results: [{ id: 'fb2' }] }) + await __flushForTesting() + expect(pushMock.mock.lastCall?.[0].requestBody.removed).toEqual(['fb2']) + expect(removalKeys()).toEqual([]) + }) + it("removes a moved session's old copy from the storage that held it, whatever its old workspace is on now", async () => { const s: Session = { id: 'mv2', name: 'session-1', createdAt: 1, workspace_id: 'ws' } sessionState.sessions = [s] diff --git a/frontend/src/lib/components/sessions/sessionMirrorPlan.ts b/frontend/src/lib/components/sessions/sessionMirrorPlan.ts index c61bcf2763..a664e61f5e 100644 --- a/frontend/src/lib/components/sessions/sessionMirrorPlan.ts +++ b/frontend/src/lib/components/sessions/sessionMirrorPlan.ts @@ -31,9 +31,9 @@ export interface MirrorSyncState { /** The user deleted the session and its removal mark could not be written to * localStorage (full): the row itself carries the removal, until it lands. */ removed?: boolean - /** The storage the push landed in, as the server names it, and the backup generation - * (bumped by a workspace key rotation) it landed under. A row recorded against another - * storage or generation describes objects the server no longer looks at. */ + /** The storage the push landed in (`storageName`), and the backup generation (bumped by + * a workspace key rotation) it landed under. A row recorded against another storage or + * generation describes objects the server no longer looks at. */ storageId?: string generation?: number /** Other storages this workspace was on that still hold a copy of the backup (a switch @@ -48,6 +48,34 @@ export interface MirrorSyncState { staging?: { chats: string[]; images: string[]; items: string[]; versions: string[] } } +const FALLBACK_STORAGE_PREFIX = 'instance:' + +/** + * How a storage the server answered from is named in the sync rows and the removal marks: + * by the id the server gives it, the instance object store standing in for a workspace + * without storage of its own (`fallback` on the answer) told apart from a workspace's own. + * A removal owed to an instance store is retired by any answer from the workspace's own + * storage (configuring one moves the backup generation past everything the workspace left + * in any instance store, so none of it is read again), where one owed to a workspace + * storage waits for that storage. + */ +export function storageName(id: string, fallback: boolean | undefined): string +export function storageName( + id: string | undefined, + fallback: boolean | undefined +): string | undefined +export function storageName( + id: string | undefined, + fallback: boolean | undefined +): string | undefined { + if (id === undefined) return undefined + return fallback ? FALLBACK_STORAGE_PREFIX + id : id +} + +export function isFallbackStorage(name: string): boolean { + return name.startsWith(FALLBACK_STORAGE_PREFIX) +} + /** * The part of a session record the backup keeps. Left out on purpose: `name` (a * per-browser counter), the unsent-draft fields (`pending_*`, `draftPrompt`, diff --git a/frontend/src/lib/components/workspaceSettings/AISettings.svelte b/frontend/src/lib/components/workspaceSettings/AISettings.svelte index 7e8b9cc76e..5bc7974af2 100644 --- a/frontend/src/lib/components/workspaceSettings/AISettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/AISettings.svelte @@ -661,7 +661,7 @@ Date: Tue, 15 Sep 2026 23:21:53 +0200 Subject: [PATCH 134/144] fix: stop reading an array job result as wm_failure or http response (#11154) * fix: only read wm_failure and wm_labels from an object job result * fix: serve an array sync result as json, not a composite response --- backend/windmill-api-jobs/src/execution.rs | 17 ++++-- backend/windmill-queue/src/jobs.rs | 52 +++++++++++++++++-- .../windmill-worker/src/result_processor.rs | 10 ++-- 3 files changed, 66 insertions(+), 13 deletions(-) diff --git a/backend/windmill-api-jobs/src/execution.rs b/backend/windmill-api-jobs/src/execution.rs index e897bf5eaf..310e44ae1f 100644 --- a/backend/windmill-api-jobs/src/execution.rs +++ b/backend/windmill-api-jobs/src/execution.rs @@ -38,8 +38,8 @@ use windmill_common::{ FlowVersionInfo, DB, }; use windmill_queue::{ - cancel_job, get_result_and_success_by_id_from_flow, push, PushArgs, PushArgsOwned, - PushIsolationLevel, + cancel_job, get_result_and_success_by_id_from_flow, parse_result_object, push, PushArgs, + PushArgsOwned, PushIsolationLevel, }; use crate::types::RunJobQuery; @@ -374,9 +374,9 @@ pub async fn run_wait_result_internal( } pub fn result_to_response(result: Box, success: bool) -> error::Result { - let composite_result = serde_json::from_str::(result.get()); + let composite_result = parse_result_object::(result.get()); match composite_result { - Ok(WindmillCompositeResult { + Some(WindmillCompositeResult { windmill_status_code, windmill_content_type, windmill_headers, @@ -1192,4 +1192,13 @@ mod result_to_response_tests { assert!(res.is_err(), "hop-by-hop header must be rejected: {name}"); } } + + #[tokio::test] + async fn array_result_is_not_a_composite_response() { + let json = r#"[201,"text/html",null,null,"

    hi

    "]"#; + let resp = result_to_response(raw(json), true).expect("response"); + + assert_eq!(resp.status(), StatusCode::OK); + assert_eq!(body_bytes(resp).await, json.as_bytes()); + } } diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index c4058dba6f..ab91d54287 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -654,6 +654,16 @@ pub struct ResultMetadata { pub wm_failure: Option, } +/// Parses a marker struct out of a job result, which only an object can carry. +/// A derived `Deserialize` also accepts an array, filling fields by position, so +/// without the check a result like `[[], "boom"]` reads as `wm_failure: "boom"`. +pub fn parse_result_object(result: &str) -> Option { + if !result.trim_start().starts_with('{') { + return None; + } + serde_json::from_str(result).ok() +} + /// Sentinel `error.name` we inject into a result when retagging a successful /// run as a failure due to `wm_failure`. Used downstream to detect that /// the result is already in the standard `{ error: { name, message }, ... }` @@ -674,8 +684,7 @@ pub fn is_pre_shaped_wm_failure_result(result: &str) -> bool { struct NameOnly { name: String, } - serde_json::from_str::(result) - .ok() + parse_result_object::(result) .and_then(|m| m.error) .map(|e| e.name == MANUAL_FAILURE_ERROR_NAME) .unwrap_or(false) @@ -721,7 +730,7 @@ impl ValidableJson for Box { } fn result_metadata(&self) -> ResultMetadata { - serde_json::from_str::(self.get()).unwrap_or_default() + parse_result_object::(self.get()).unwrap_or_default() } fn size(&self) -> usize { @@ -774,6 +783,10 @@ impl ValidableJson for serde_json::Value { } fn result_metadata(&self) -> ResultMetadata { + // An array would decode positionally, see `parse_result_object`. + if !self.is_object() { + return ResultMetadata::default(); + } serde_json::from_value::(self.clone()).unwrap_or_default() } @@ -7876,3 +7889,36 @@ mod git_sync_concurrency_key_tests { assert!(a.len() <= 255 && b.len() <= 255); } } + +#[cfg(test)] +mod result_metadata_tests { + use super::{ResultMetadata, ValidableJson}; + use serde_json::value::RawValue; + + fn from_raw(json: &str) -> ResultMetadata { + RawValue::from_string(json.to_string()) + .unwrap() + .result_metadata() + } + + fn from_value(json: &str) -> ResultMetadata { + serde_json::from_str::(json) + .unwrap() + .result_metadata() + } + + #[test] + fn array_result_carries_no_markers() { + for json in [r#"[["label"], "boom"]"#, r#"[null, "boom"]"#] { + for meta in [from_raw(json), from_value(json)] { + assert!( + meta.wm_labels.is_none() && meta.wm_failure.is_none(), + "{json}" + ); + } + } + let meta = from_raw(r#"{"wm_labels": ["label"], "wm_failure": "boom"}"#); + assert_eq!(meta.wm_labels, Some(vec!["label".to_string()])); + assert_eq!(meta.wm_failure.as_deref(), Some("boom")); + } +} diff --git a/backend/windmill-worker/src/result_processor.rs b/backend/windmill-worker/src/result_processor.rs index a269e72f46..1529d474fa 100644 --- a/backend/windmill-worker/src/result_processor.rs +++ b/backend/windmill-worker/src/result_processor.rs @@ -32,8 +32,8 @@ use windmill_common::bench::{BenchmarkInfo, BenchmarkIter}; use windmill_queue::{ append_logs, asset_dispatch, get_mini_completed_job, is_pre_shaped_wm_failure_result, - CanceledBy, FlowRunners, JobCompleted, MiniCompletedJob, MiniPulledJob, ValidableJson, - WrappedError, INIT_SCRIPT_TAG, MANUAL_FAILURE_ERROR_NAME, + parse_result_object, CanceledBy, FlowRunners, JobCompleted, MiniCompletedJob, MiniPulledJob, + ValidableJson, WrappedError, INIT_SCRIPT_TAG, MANUAL_FAILURE_ERROR_NAME, }; use serde_json::{json, value::RawValue, Value}; @@ -72,13 +72,11 @@ struct NestedErrorMessage { /// named `name`/`message`), and we want OTel to record the ManualFailure /// rather than the user's sibling fields. fn extract_error_message(raw: &str) -> Option { - let nested = serde_json::from_str::(raw) - .ok() - .map(|n| n.error); + let nested = parse_result_object::(raw).map(|n| n.error); if matches!(&nested, Some(em) if em.name == MANUAL_FAILURE_ERROR_NAME) { return nested; } - if let Ok(em) = serde_json::from_str::(raw) { + if let Some(em) = parse_result_object::(raw) { return Some(em); } nested From d0d5b295b9942621d3bbf705f0c9e3b8e8637d68 Mon Sep 17 00:00:00 2001 From: AlexRV12 <71396855+AlexRV12@users.noreply.github.com> Date: Wed, 16 Sep 2026 00:26:16 +0200 Subject: [PATCH 135/144] docs: terminate datatable write examples with .execute() (#11148) Co-authored-by: Claude Opus 5 (1M context) --- cli/src/guidance/skills.gen.ts | 23 ++++++++++--------- system_prompts/auto-generated/prompts.ts | 23 ++++++++++--------- .../auto-generated/skills/raw-app/SKILL.md | 23 ++++++++++--------- system_prompts/base/raw-app.md | 23 ++++++++++--------- 4 files changed, 48 insertions(+), 44 deletions(-) diff --git a/cli/src/guidance/skills.gen.ts b/cli/src/guidance/skills.gen.ts index acedfa982b..60fdd07124 100644 --- a/cli/src/guidance/skills.gen.ts +++ b/cli/src/guidance/skills.gen.ts @@ -6012,8 +6012,8 @@ export async function main(user_id: string) { const users = await sql\`SELECT * FROM users WHERE active = \${true}\`.fetch(); // Insert/Update - await sql\`INSERT INTO users (name, email) VALUES (\${name}, \${email})\`; - await sql\`UPDATE users SET name = \${newName} WHERE id = \${user_id}\`; + await sql\`INSERT INTO users (name, email) VALUES (\${name}, \${email})\`.execute(); + await sql\`UPDATE users SET name = \${newName} WHERE id = \${user_id}\`.execute(); return user; } @@ -6032,8 +6032,8 @@ def main(user_id: str): users = db.query('SELECT * FROM users WHERE active = $1', True).fetch() # Insert/Update - db.query('INSERT INTO users (name, email) VALUES ($1, $2)', name, email) - db.query('UPDATE users SET name = $1 WHERE id = $2', new_name, user_id) + db.query('INSERT INTO users (name, email) VALUES ($1, $2)', name, email).execute() + db.query('UPDATE users SET name = $1 WHERE id = $2', new_name, user_id).execute() return user \`\`\` @@ -6042,13 +6042,14 @@ def main(user_id: str): 1. **Check existing tables** before creating new ones — reuse beats schema growth. 2. **Use parameterized queries** — never concatenate user input into SQL. -3. **Keep runnables focused** — one function per runnable; small surface area. -4. **Use descriptive keys** — \`get_user\`, not \`a\`. -5. **Always whitelist tables** — adding a runnable that queries a new table requires the table to be in \`data.tables\` first. -6. **Mark sensitive UI with \`data-wm-no-record\`** — it is what keeps that data out of a recorded demo; passwords are handled for you. -7. **Reach for \`backendAsync\` + \`waitJob\`** for long work — never a hand-written job-polling runnable. -8. **Deploy what a path runnable points at** — a path runnable aimed at a draft fails at runtime; tell the user what needs deploying. -9. **Use \`windmill-chat\` for a chat over a chat-mode flow** — never a runnable that runs the flow and polls its stream. +3. **Terminate every datatable statement** — the tagged template and \`db.query(...)\` only build a statement. It runs when you call \`fetch\` / \`fetchOne\` / \`fetchOneScalar\` / \`execute\` (\`fetch\` / \`fetch_one\` / \`fetch_one_scalar\` / \`execute\` in Python). An INSERT or UPDATE without one writes nothing and raises nothing. Awaiting the statement itself is a no-op — it is not a promise. +4. **Keep runnables focused** — one function per runnable; small surface area. +5. **Use descriptive keys** — \`get_user\`, not \`a\`. +6. **Always whitelist tables** — adding a runnable that queries a new table requires the table to be in \`data.tables\` first. +7. **Mark sensitive UI with \`data-wm-no-record\`** — it is what keeps that data out of a recorded demo; passwords are handled for you. +8. **Reach for \`backendAsync\` + \`waitJob\`** for long work — never a hand-written job-polling runnable. +9. **Deploy what a path runnable points at** — a path runnable aimed at a draft fails at runtime; tell the user what needs deploying. +10. **Use \`windmill-chat\` for a chat over a chat-mode flow** — never a runnable that runs the flow and polls its stream. `, "triggers": `--- name: triggers diff --git a/system_prompts/auto-generated/prompts.ts b/system_prompts/auto-generated/prompts.ts index 4b4552dca4..2a20e8f779 100644 --- a/system_prompts/auto-generated/prompts.ts +++ b/system_prompts/auto-generated/prompts.ts @@ -920,8 +920,8 @@ export async function main(user_id: string) { const users = await sql\`SELECT * FROM users WHERE active = \${true}\`.fetch(); // Insert/Update - await sql\`INSERT INTO users (name, email) VALUES (\${name}, \${email})\`; - await sql\`UPDATE users SET name = \${newName} WHERE id = \${user_id}\`; + await sql\`INSERT INTO users (name, email) VALUES (\${name}, \${email})\`.execute(); + await sql\`UPDATE users SET name = \${newName} WHERE id = \${user_id}\`.execute(); return user; } @@ -940,8 +940,8 @@ def main(user_id: str): users = db.query('SELECT * FROM users WHERE active = $1', True).fetch() # Insert/Update - db.query('INSERT INTO users (name, email) VALUES ($1, $2)', name, email) - db.query('UPDATE users SET name = $1 WHERE id = $2', new_name, user_id) + db.query('INSERT INTO users (name, email) VALUES ($1, $2)', name, email).execute() + db.query('UPDATE users SET name = $1 WHERE id = $2', new_name, user_id).execute() return user \`\`\` @@ -950,13 +950,14 @@ def main(user_id: str): 1. **Check existing tables** before creating new ones — reuse beats schema growth. 2. **Use parameterized queries** — never concatenate user input into SQL. -3. **Keep runnables focused** — one function per runnable; small surface area. -4. **Use descriptive keys** — \`get_user\`, not \`a\`. -5. **Always whitelist tables** — adding a runnable that queries a new table requires the table to be in \`data.tables\` first. -6. **Mark sensitive UI with \`data-wm-no-record\`** — it is what keeps that data out of a recorded demo; passwords are handled for you. -7. **Reach for \`backendAsync\` + \`waitJob\`** for long work — never a hand-written job-polling runnable. -8. **Deploy what a path runnable points at** — a path runnable aimed at a draft fails at runtime; tell the user what needs deploying. -9. **Use \`windmill-chat\` for a chat over a chat-mode flow** — never a runnable that runs the flow and polls its stream. +3. **Terminate every datatable statement** — the tagged template and \`db.query(...)\` only build a statement. It runs when you call \`fetch\` / \`fetchOne\` / \`fetchOneScalar\` / \`execute\` (\`fetch\` / \`fetch_one\` / \`fetch_one_scalar\` / \`execute\` in Python). An INSERT or UPDATE without one writes nothing and raises nothing. Awaiting the statement itself is a no-op — it is not a promise. +4. **Keep runnables focused** — one function per runnable; small surface area. +5. **Use descriptive keys** — \`get_user\`, not \`a\`. +6. **Always whitelist tables** — adding a runnable that queries a new table requires the table to be in \`data.tables\` first. +7. **Mark sensitive UI with \`data-wm-no-record\`** — it is what keeps that data out of a recorded demo; passwords are handled for you. +8. **Reach for \`backendAsync\` + \`waitJob\`** for long work — never a hand-written job-polling runnable. +9. **Deploy what a path runnable points at** — a path runnable aimed at a draft fails at runtime; tell the user what needs deploying. +10. **Use \`windmill-chat\` for a chat over a chat-mode flow** — never a runnable that runs the flow and polls its stream. `; export const PIPELINE_BASE = `# Data pipeline authoring diff --git a/system_prompts/auto-generated/skills/raw-app/SKILL.md b/system_prompts/auto-generated/skills/raw-app/SKILL.md index da4a9729be..ed79172b9d 100644 --- a/system_prompts/auto-generated/skills/raw-app/SKILL.md +++ b/system_prompts/auto-generated/skills/raw-app/SKILL.md @@ -435,8 +435,8 @@ export async function main(user_id: string) { const users = await sql`SELECT * FROM users WHERE active = ${true}`.fetch(); // Insert/Update - await sql`INSERT INTO users (name, email) VALUES (${name}, ${email})`; - await sql`UPDATE users SET name = ${newName} WHERE id = ${user_id}`; + await sql`INSERT INTO users (name, email) VALUES (${name}, ${email})`.execute(); + await sql`UPDATE users SET name = ${newName} WHERE id = ${user_id}`.execute(); return user; } @@ -455,8 +455,8 @@ def main(user_id: str): users = db.query('SELECT * FROM users WHERE active = $1', True).fetch() # Insert/Update - db.query('INSERT INTO users (name, email) VALUES ($1, $2)', name, email) - db.query('UPDATE users SET name = $1 WHERE id = $2', new_name, user_id) + db.query('INSERT INTO users (name, email) VALUES ($1, $2)', name, email).execute() + db.query('UPDATE users SET name = $1 WHERE id = $2', new_name, user_id).execute() return user ``` @@ -465,10 +465,11 @@ def main(user_id: str): 1. **Check existing tables** before creating new ones — reuse beats schema growth. 2. **Use parameterized queries** — never concatenate user input into SQL. -3. **Keep runnables focused** — one function per runnable; small surface area. -4. **Use descriptive keys** — `get_user`, not `a`. -5. **Always whitelist tables** — adding a runnable that queries a new table requires the table to be in `data.tables` first. -6. **Mark sensitive UI with `data-wm-no-record`** — it is what keeps that data out of a recorded demo; passwords are handled for you. -7. **Reach for `backendAsync` + `waitJob`** for long work — never a hand-written job-polling runnable. -8. **Deploy what a path runnable points at** — a path runnable aimed at a draft fails at runtime; tell the user what needs deploying. -9. **Use `windmill-chat` for a chat over a chat-mode flow** — never a runnable that runs the flow and polls its stream. +3. **Terminate every datatable statement** — the tagged template and `db.query(...)` only build a statement. It runs when you call `fetch` / `fetchOne` / `fetchOneScalar` / `execute` (`fetch` / `fetch_one` / `fetch_one_scalar` / `execute` in Python). An INSERT or UPDATE without one writes nothing and raises nothing. Awaiting the statement itself is a no-op — it is not a promise. +4. **Keep runnables focused** — one function per runnable; small surface area. +5. **Use descriptive keys** — `get_user`, not `a`. +6. **Always whitelist tables** — adding a runnable that queries a new table requires the table to be in `data.tables` first. +7. **Mark sensitive UI with `data-wm-no-record`** — it is what keeps that data out of a recorded demo; passwords are handled for you. +8. **Reach for `backendAsync` + `waitJob`** for long work — never a hand-written job-polling runnable. +9. **Deploy what a path runnable points at** — a path runnable aimed at a draft fails at runtime; tell the user what needs deploying. +10. **Use `windmill-chat` for a chat over a chat-mode flow** — never a runnable that runs the flow and polls its stream. diff --git a/system_prompts/base/raw-app.md b/system_prompts/base/raw-app.md index 3849a465a0..f7eaef944d 100644 --- a/system_prompts/base/raw-app.md +++ b/system_prompts/base/raw-app.md @@ -200,8 +200,8 @@ export async function main(user_id: string) { const users = await sql`SELECT * FROM users WHERE active = ${true}`.fetch(); // Insert/Update - await sql`INSERT INTO users (name, email) VALUES (${name}, ${email})`; - await sql`UPDATE users SET name = ${newName} WHERE id = ${user_id}`; + await sql`INSERT INTO users (name, email) VALUES (${name}, ${email})`.execute(); + await sql`UPDATE users SET name = ${newName} WHERE id = ${user_id}`.execute(); return user; } @@ -220,8 +220,8 @@ def main(user_id: str): users = db.query('SELECT * FROM users WHERE active = $1', True).fetch() # Insert/Update - db.query('INSERT INTO users (name, email) VALUES ($1, $2)', name, email) - db.query('UPDATE users SET name = $1 WHERE id = $2', new_name, user_id) + db.query('INSERT INTO users (name, email) VALUES ($1, $2)', name, email).execute() + db.query('UPDATE users SET name = $1 WHERE id = $2', new_name, user_id).execute() return user ``` @@ -230,10 +230,11 @@ def main(user_id: str): 1. **Check existing tables** before creating new ones — reuse beats schema growth. 2. **Use parameterized queries** — never concatenate user input into SQL. -3. **Keep runnables focused** — one function per runnable; small surface area. -4. **Use descriptive keys** — `get_user`, not `a`. -5. **Always whitelist tables** — adding a runnable that queries a new table requires the table to be in `data.tables` first. -6. **Mark sensitive UI with `data-wm-no-record`** — it is what keeps that data out of a recorded demo; passwords are handled for you. -7. **Reach for `backendAsync` + `waitJob`** for long work — never a hand-written job-polling runnable. -8. **Deploy what a path runnable points at** — a path runnable aimed at a draft fails at runtime; tell the user what needs deploying. -9. **Use `windmill-chat` for a chat over a chat-mode flow** — never a runnable that runs the flow and polls its stream. +3. **Terminate every datatable statement** — the tagged template and `db.query(...)` only build a statement. It runs when you call `fetch` / `fetchOne` / `fetchOneScalar` / `execute` (`fetch` / `fetch_one` / `fetch_one_scalar` / `execute` in Python). An INSERT or UPDATE without one writes nothing and raises nothing. Awaiting the statement itself is a no-op — it is not a promise. +4. **Keep runnables focused** — one function per runnable; small surface area. +5. **Use descriptive keys** — `get_user`, not `a`. +6. **Always whitelist tables** — adding a runnable that queries a new table requires the table to be in `data.tables` first. +7. **Mark sensitive UI with `data-wm-no-record`** — it is what keeps that data out of a recorded demo; passwords are handled for you. +8. **Reach for `backendAsync` + `waitJob`** for long work — never a hand-written job-polling runnable. +9. **Deploy what a path runnable points at** — a path runnable aimed at a draft fails at runtime; tell the user what needs deploying. +10. **Use `windmill-chat` for a chat over a chat-mode flow** — never a runnable that runs the flow and polls its stream. From a9a9335a34a13ffd8cd2699adc92087b679548ca Mon Sep 17 00:00:00 2001 From: Guilhem Date: Wed, 16 Sep 2026 00:26:56 +0200 Subject: [PATCH 136/144] fix: keep the instance users table's actions and header in view (#11145) * fix: keep the instance users table's actions and header in view Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_015pMjimm9tUMXtD8rkWB62w * fix: lock only the User option for group-granted roles and keep pinned cells opaque Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_015pMjimm9tUMXtD8rkWB62w * fix: close the instance settings drawer from the manage-in-workspace menu item Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_015pMjimm9tUMXtD8rkWB62w --------- Co-authored-by: Claude Fable 5.1 Co-authored-by: Ruben Fiszel --- .../components/ChangeInstanceUsername.svelte | 20 +- .../lib/components/SuperadminSettings.svelte | 2 +- .../components/SuperadminSettingsInner.svelte | 326 ++++++++++-------- .../toggleButton-v2/ToggleButton.svelte | 14 +- .../settings/ForkMemberSettings.svelte | 4 +- .../settings/WorkspaceUserSettings.svelte | 8 +- frontend/src/lib/components/table/Head.svelte | 11 +- 7 files changed, 232 insertions(+), 153 deletions(-) diff --git a/frontend/src/lib/components/ChangeInstanceUsername.svelte b/frontend/src/lib/components/ChangeInstanceUsername.svelte index 1fcd726cea..fce0f4e8f9 100644 --- a/frontend/src/lib/components/ChangeInstanceUsername.svelte +++ b/frontend/src/lib/components/ChangeInstanceUsername.svelte @@ -3,6 +3,7 @@ import Popover from './meltComponents/Popover.svelte' import { autoPlacement } from '@floating-ui/core' import ChangeInstanceUsernameInner from './ChangeInstanceUsernameInner.svelte' + import { AlertTriangle } from 'lucide-svelte' interface Props { email: string @@ -24,9 +25,22 @@ closeButton > {#snippet trigger()} - + {#if isConflict} + + + {/if} {/snippet} {#snippet content()} - + {#snippet titleExtra()} diff --git a/frontend/src/lib/components/SuperadminSettingsInner.svelte b/frontend/src/lib/components/SuperadminSettingsInner.svelte index 97b15fe443..0fd7991597 100644 --- a/frontend/src/lib/components/SuperadminSettingsInner.svelte +++ b/frontend/src/lib/components/SuperadminSettingsInner.svelte @@ -26,11 +26,10 @@ CheckCircle2, ExternalLink, Pencil, + Settings, UserMinus, UserPlus } from 'lucide-svelte' - import Badge from './common/badge/Badge.svelte' - import Tooltip from './Tooltip.svelte' import DropdownV2 from './DropdownV2.svelte' import Popover from './meltComponents/Popover.svelte' import ConfirmationModal from './common/confirmationModal/ConfirmationModal.svelte' @@ -155,6 +154,11 @@ loadExtJwtPage(1) let tab: string = $state('users') + let usersListShown = $derived( + tab === 'users' && + !yamlMode && + (usersSubTab === 'users' || (usersSubTab === 'ext_jwt' && extJwtTokens.length === 0)) + ) $effect(() => { tab = $instanceSettingsSelectedTab @@ -320,11 +324,14 @@
    -
    + +
    {#if tab === 'ai' && !yamlMode} {:else if tab === 'users' && !yamlMode} -
    +
    {#if !automateUsernameCreation && !isCloudHosted()}

    Automatic username creation

    @@ -373,7 +380,7 @@ - {#if usersSubTab === 'users' || (usersSubTab === 'ext_jwt' && extJwtTokens.length === 0)} + {#if usersListShown} {filteredUsers.length} user{filteredUsers.length !== 1 ? 's' : ''} found

    -
    - 50} - loadMore={50} - on:loadMore={() => { - nbDisplayed += 50 - }} - > + +
    +
    • Email @@ -434,7 +437,7 @@ Kind {/if} Role - + Actions @@ -443,12 +446,22 @@ {#if filteredUsers && users} {#each filteredUsers.slice(0, nbDisplayed) as { email, super_admin, devops, login_type, name, username, operator_only, is_workspace_admin, role_source, disabled, workspace_id }, i (email + '::' + (workspace_id ?? ''))} {@const isServiceAccount = login_type === 'service_account'} + {@const groupRole = + role_source === 'instance_group' && (super_admin || devops)} + + {@const groupRoleTooltip = + 'Role is set by an instance group. Superadmin and Devops can be set here, but demoting to User requires removing the user from the group.'} + {@const serviceAccountTooltip = + 'Service accounts are always users in the instance. Their workspace role is managed in the workspace user settings.'} + - +
    {#if isServiceAccount} @@ -458,14 +471,6 @@ >{email} {/if} - {#if workspace_id} - - {truncate(workspace_id, 20)} - - {/if} {#if disabled} {#if automateUsernameCreation} - + {#if username} {username} {:else} @@ -503,133 +508,157 @@ > {#if activeOnly} - {#if is_workspace_admin} - Admin - {:else if operator_only} - Operator only - {:else} - Developer - {/if} + + {#if is_workspace_admin} + Admin + {:else if operator_only} + Operator only + {:else} + Developer + {/if} + {/if} - {#if isServiceAccount} -
    + +
    + {#key `${super_admin}_${devops}_${role_source}`} + { + if (email == $userStore?.email) { + sendUserToast('You cannot demote yourself', true) + listUsers(activeOnly) + return + } + + let role = e.detail + + if (role === 'super_admin') { + await UserService.globalUserUpdate({ + email, + requestBody: { + is_super_admin: true, + is_devops: false + } + }) + } + if (role === 'devops') { + await UserService.globalUserUpdate({ + email, + requestBody: { + is_super_admin: false, + is_devops: true + } + }) + } + if (role === 'user') { + await UserService.globalUserUpdate({ + email, + requestBody: { + is_super_admin: false, + is_devops: false + } + }) + } + sendUserToast('User updated') + listUsers(activeOnly) + }} + > + {#snippet children({ item })} + + + + {/snippet} + + {/key} + {#if isServiceAccount} {is_workspace_admin ? 'Admin' : operator_only ? 'Operator' : 'Developer'} + in + {#if workspace_id} + closeDrawer?.()} + >{truncate(workspace_id, 20)} + {:else} + its workspace + {/if} - - Service-account role is managed in the workspace user settings. - -
    - {:else} -
    - {#key `${super_admin}_${devops}_${role_source}`} - { - if (email == $userStore?.email) { - sendUserToast('You cannot demote yourself', true) - listUsers(activeOnly) - return - } - - let role = e.detail - - if (role === 'super_admin') { - await UserService.globalUserUpdate({ - email, - requestBody: { - is_super_admin: true, - is_devops: false - } - }) - } - if (role === 'devops') { - await UserService.globalUserUpdate({ - email, - requestBody: { - is_super_admin: false, - is_devops: true - } - }) - } - if (role === 'user') { - await UserService.globalUserUpdate({ - email, - requestBody: { - is_super_admin: false, - is_devops: false - } - }) - } - sendUserToast('User updated') - listUsers(activeOnly) - }} - > - {#snippet children({ item })} - - - - {/snippet} - - {/key} - {#if role_source === 'instance_group' && (super_admin || devops)} - closeDrawer?.()} - > - Set by instance group - - {/if} -
    - {/if} + {:else if groupRole} + closeDrawer?.()} + > + Set by instance group + + {/if} +
    - +
    {#if isServiceAccount} {#if workspace_id} - Manage in workspace + closeDrawer?.(), + href: `${base}/workspace_settings?tab=users&workspace=${workspace_id}` + } + ]} + /> {/if} {:else}
    {/each} + {#if filteredUsers.length > nbDisplayed} + {@const remaining = Math.min(50, filteredUsers.length - nbDisplayed)} + + + + + + + {/if} {/if} diff --git a/frontend/src/lib/components/common/toggleButton-v2/ToggleButton.svelte b/frontend/src/lib/components/common/toggleButton-v2/ToggleButton.svelte index 621e8383d0..2773503df9 100644 --- a/frontend/src/lib/components/common/toggleButton-v2/ToggleButton.svelte +++ b/frontend/src/lib/components/common/toggleButton-v2/ToggleButton.svelte @@ -7,6 +7,9 @@ interface Props { label?: string | undefined + /** Shown instead of `label` below the `xl` breakpoint, for groups that must keep + * their width inside a narrow table cell. The full label stays the accessible name. */ + shortLabel?: string | undefined iconOnly?: boolean tooltip?: string | undefined icon?: any | undefined @@ -30,6 +33,7 @@ let { label = undefined, + shortLabel = undefined, iconOnly = false, tooltip = undefined, icon = undefined, @@ -68,6 +72,7 @@ @@ -187,7 +187,7 @@ {is_admin ? 'Admin' : operator ? 'Operator' : 'Developer'} - + @@ -1142,7 +1142,7 @@ - + + + {@render children?.()} {@render headerAction?.()} From ee6d317e318fa8a1506fb18d51c45b627070627a Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 16 Sep 2026 09:27:49 +0200 Subject: [PATCH 137/144] feat: retention for AI sessions on the object store and in the browser (#11152) * feat: retention for AI sessions, swept on the object store and in the browser Co-Authored-By: Claude Fable 5.1 * fix: make the retention sweeps retryable and safe against pushes Co-Authored-By: Claude Opus 5 (1M context) * fix: spare other tabs' sessions, reclaim abandoned split pushes Co-Authored-By: Claude Opus 5 (1M context) * fix: sweep under an exclusive session lock, keep the captured user Co-Authored-By: Claude Opus 5 (1M context) * docs: say a tab selecting a session mid-sweep is not held back Co-Authored-By: Claude Opus 5 (1M context) * fix: sweep local sessions only while no other tab has them loaded Co-Authored-By: Claude Opus 5 (1M context) * fix: one tab sweeps at a time, and keeps the switched user's hold Co-Authored-By: Claude Opus 5 (1M context) * test: push the fallback session again before the rotation assertions Co-Authored-By: Claude Opus 5 (1M context) * refactor: keep retention server-side here, move the browser sweep out Co-Authored-By: Claude Opus 5 (1M context) * docs: the retention setting no longer touches browser-local sessions Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Fable 5.1 --- backend/src/monitor.rs | 18 + backend/tests/ai_sessions.rs | 172 +++++++++ .../src/ai_session_backups.rs | 5 +- .../windmill-api-workspaces/src/workspaces.rs | 8 + backend/windmill-api/openapi.yaml | 8 + backend/windmill-api/src/ai.rs | 20 ++ backend/windmill-api/src/ai_sessions.rs | 333 ++++++++++++++++-- backend/windmill-api/src/lib.rs | 2 + backend/windmill-api/src/workspaces.rs | 7 + docs/ai-session-backups.md | 34 +- .../workspaceSettings/AISettings.svelte | 57 ++- 11 files changed, 628 insertions(+), 36 deletions(-) diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 493abd172e..dc3b9b6e8b 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -4380,6 +4380,23 @@ pub async fn monitor_db( } }; + // Delete the AI session backups older than their workspace's retention. Every ~40 min + // (240 iterations at the default 10 s, the most a u8 `should_run` counts): the retention + // counts in days. Spawned for the same reason as the credential maintenance above, a + // sweep of many sessions outlasting the join's deadline; the sweep's own advisory lock + // keeps one server at a time at it. + let ai_session_retention_f = async { + #[cfg(feature = "parquet")] + if server_mode && iteration.is_some() && iteration.as_ref().unwrap().should_run(240) { + if let Some(db) = conn.as_sql() { + let db = db.clone(); + tokio::spawn( + async move { windmill_api::sweep_expired_ai_session_backups(&db).await }, + ); + } + } + }; + // run every 2 iterations (~20s at the default LISTEN_NEW_EVENTS_INTERVAL_SEC). // Enterprise feature: the active `// freshness` backstop lives in // windmill-queue's `freshness_watchdog` (`private`); OSS gets a no-op stub. @@ -4434,6 +4451,7 @@ pub async fn monitor_db( cleanup_scheduled_job_deletions_f, git_auto_pull_f, git_credential_maintenance_f, + ai_session_retention_f, pipeline_freshness_watchdog_f, reconcile_unarmed_schedules_f, ); diff --git a/backend/tests/ai_sessions.rs b/backend/tests/ai_sessions.rs index 3eb28cfae8..6b33a4141e 100644 --- a/backend/tests/ai_sessions.rs +++ b/backend/tests/ai_sessions.rs @@ -1235,6 +1235,161 @@ async fn test_backup_writes_are_refused_for_the_wrong_owner_token_or_id( Ok(()) } +/// Sets the object's modification time `days` back: the FilesystemStorage answers +/// `last_modified` from it, so this is a session no push touched since. +fn age_object(path: &std::path::Path, days: u64) -> std::io::Result<()> { + let at = std::time::SystemTime::now() - std::time::Duration::from_secs(days * 86_400); + std::fs::File::options() + .write(true) + .open(path)? + .set_modified(at) +} + +#[sqlx::test(fixtures("base"))] +async fn test_expired_backups_are_swept_by_age_and_left_out_of_the_listing( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let base = format!( + "http://localhost:{}/api/w/test-workspace", + server.addr.port() + ); + let storage_dir = tempfile::tempdir()?; + configure_primary_lfs(&db, &storage_dir.path().to_string_lossy()).await?; + + let chat = |sid: &str, cid: &str| { + json!({ "id": cid, "record": { "id": cid, "sessionId": sid, "lastModified": 2, + "actualMessages": [], "displayMessages": [] } }) + }; + let whole = |sid: &str| { + json!({ + "id": sid, "whole": true, "epoch": 0, + "head": { "id": sid, "workspace_id": "test-workspace", "createdAt": 1, "chatId": "c1" }, + "chats": [chat(sid, "c1")], + "images": [{ "chat_id": "c1", "id": "img1", "data_url": "data:image/png;base64,AAAA" }], + "artifacts": { "items": [], "versions": [] } + }) + }; + // Two pushes split over parts of which only the first part landed: one a browser + // abandoned long ago (its token aged past the retention), one still in flight. + let opening = |sid: &str| { + json!({ + "id": sid, "whole": true, "epoch": 0, "push": format!("t-{sid}"), "opens": true, + "partial": true, "chats": [chat(sid, "c1")], + "head": { "id": sid, "workspace_id": "test-workspace", "createdAt": 1, "chatId": "c1" } + }) + }; + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "sessions": [ + whole("old"), whole("live"), opening("abandoned"), opening("inflight") + ] }), + ) + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let root = user_root(storage_dir.path(), "test@windmill.dev"); + age_object(&root.join("index/old/0"), 40)?; + age_object(&root.join("index/abandoned/push"), 40)?; + + let listed = |listing: Value| -> Vec { + let mut ids: Vec = listing["sessions"] + .as_array() + .unwrap() + .iter() + .map(|s| s["id"].as_str().unwrap().to_string()) + .collect(); + ids.sort(); + ids + }; + let objects = |root: &std::path::Path| -> Vec { + files_under(root) + .into_iter() + .map(|(p, _)| p.strip_prefix(root).unwrap().to_string_lossy().into_owned()) + .collect() + }; + + // Without a retention nothing is swept, however old. + windmill_api::sweep_expired_ai_session_backups(&db).await; + assert_eq!(listed(list(&base, "SECRET_TOKEN").await?), ["live", "old"]); + + let set_retention = |days: Value| { + authed( + client().post(format!("{base}/workspaces/edit_copilot_config")), + "SECRET_TOKEN", + ) + .json(&json!({ "sessions_retention_days": days })) + .send() + }; + let resp = set_retention(json!(0)).await?; + assert_eq!(resp.status(), 400, "{}", resp.text().await?); + let resp = set_retention(json!(30)).await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + + // The listing leaves the expired session out before the sweep reaches it. + assert_eq!(listed(list(&base, "SECRET_TOKEN").await?), ["live"]); + assert!(root.join("sessions/old/head.json").exists()); + + // A removal cut short (a directory stands where the head is, so it cannot be unlinked) + // leaves the sweep's record with the markers gone; the next pass finds it and finishes. + let head = root.join("sessions/old/head.json"); + std::fs::remove_file(&head)?; + std::fs::create_dir(&head)?; + std::fs::write(head.join("planted"), b"")?; + windmill_api::sweep_expired_ai_session_backups(&db).await; + assert!(root.join("index/old/sweep").exists()); + assert!(!root.join("index/old/0").exists()); + assert!(root.join("sessions/old/chats/c1.json").exists()); + assert_eq!(listed(list(&base, "SECRET_TOKEN").await?), ["live"]); + std::fs::remove_dir_all(&head)?; + + windmill_api::sweep_expired_ai_session_backups(&db).await; + let remaining = objects(&root); + assert!( + remaining + .iter() + .all(|p| !p.contains("/old/") && !p.contains("/abandoned/")), + "{remaining:?}" + ); + for kept in [ + "index/live/0", + "sessions/live/head.json", + "sessions/live/chats/c1.json", + "images/live/c1/img1", + "index/inflight/push", + "sessions/inflight/head.json", + ] { + assert!( + remaining.iter().any(|p| p == kept), + "{kept} in {remaining:?}" + ); + } + assert_eq!(listed(list(&base, "SECRET_TOKEN").await?), ["live"]); + + // A second pass has nothing to do; a session pushed again since its marker aged is + // renewed by the push, which rewrites the marker. + windmill_api::sweep_expired_ai_session_backups(&db).await; + age_object(&root.join("index/live/0"), 40)?; + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", + "sessions": [{ "id": "live", "epoch": 0, "chats": [chat("live", "c2")] }] }), + ) + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + windmill_api::sweep_expired_ai_session_backups(&db).await; + let mut after = objects(&root); + after.sort(); + let mut expected = remaining.clone(); + expected.push("sessions/live/chats/c2.json".to_string()); + expected.sort(); + assert_eq!(after, expected); + assert_eq!(listed(list(&base, "SECRET_TOKEN").await?), ["live"]); + Ok(()) +} + /// Puts the process-wide instance store back to none, even when an assertion fails. struct ResetInstanceStore; impl Drop for ResetInstanceStore { @@ -1333,6 +1488,23 @@ async fn test_backups_fall_back_to_the_instance_storage(db: Pool) -> a .unwrap_or_else(|| panic!("no fallback usage in {usage}")); assert!(fallback_usage["bytes"].as_i64().unwrap() > 0); + // The retention sweep reaches what the instance store keeps for the workspace, choosing + // that store from the row it reads the generation from. + let instance_user = user_root(instance_dir.path(), "test@windmill.dev"); + age_object(&instance_user.join("index/s1/0"), 40)?; + sqlx::query( + "UPDATE workspace_settings SET ai_config = '{\"sessions_retention_days\": 30}' \ + WHERE workspace_id = 'test-workspace'", + ) + .execute(&db) + .await?; + windmill_api::sweep_expired_ai_session_backups(&db).await; + assert!(!instance_user.join("index/s1/0").exists()); + assert!(!instance_user.join("sessions/s1/head.json").exists()); + // Pushed again, so the rotation below has a backup to delete. + assert_eq!(push_whole().await?.status(), 200); + assert!(!files_under(&in_instance).is_empty()); + // A key rotation sweeps the older generation out of the instance store too. rotate(&base, &"c".repeat(64)).await?; wait_until_empty(&in_instance, "a rotation on the instance store").await; diff --git a/backend/windmill-api-workspaces/src/ai_session_backups.rs b/backend/windmill-api-workspaces/src/ai_session_backups.rs index 46c91d5354..8e7b75e1e2 100644 --- a/backend/windmill-api-workspaces/src/ai_session_backups.rs +++ b/backend/windmill-api-workspaces/src/ai_session_backups.rs @@ -123,8 +123,9 @@ pub async fn fallback_store(db: &DB) -> Result> { } /// The workspace's primary storage, resolved without a caller: a rotation runs its deletion -/// off its own request. -async fn primary_store(db: &DB, w_id: &str) -> Result> { +/// off its own request, and the retention sweep off any. The caller must be the server +/// itself; nothing here checks who asks. +pub async fn primary_store(db: &DB, w_id: &str) -> Result> { let Some(lfs_json) = sqlx::query_scalar!( "SELECT large_file_storage FROM workspace_settings WHERE workspace_id = $1", w_id diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index c240948105..c29d99d05b 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -5629,6 +5629,14 @@ struct SessionWorkspaceStatusRequest { workspace_ids: Vec, } +/// `ai_config.sessions_retention_days` as stored, `None` when unset or not a count of days. +pub fn sessions_retention_days(value: Option<&serde_json::Value>) -> Option { + value + .and_then(|v| v.as_u64()) + .filter(|days| *days >= 1) + .and_then(|days| u32::try_from(days).ok()) +} + /// Reconciliation support for client-side AI sessions, which the backend cannot touch /// directly. The client posts the workspace ids its sessions reference and uses the /// per-id status to keep sessions in sync with workspace lifecycle: `deleted` (no row, or diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 63e8dd46ea..9ce1aef272 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -28448,6 +28448,14 @@ components: description: >- Stops browsers from backing their AI sessions up to the workspace's object storage. Read from the workspace's own settings like `copilot_disabled`. + sessions_retention_days: + type: integer + minimum: 1 + maximum: 3650 + description: >- + The server deletes the backup of a session no push has reached for this many + days. Unset keeps backups until the user deletes the session. Read from the + workspace's own settings like `copilot_disabled`. AISessionBackupListing: type: object diff --git a/backend/windmill-api/src/ai.rs b/backend/windmill-api/src/ai.rs index 8b9c7929ab..1710db9cec 100644 --- a/backend/windmill-api/src/ai.rs +++ b/backend/windmill-api/src/ai.rs @@ -455,6 +455,12 @@ pub struct AIConfig { /// (`ai_sessions.rs`). Read from the workspace's own row like `copilot_disabled`. #[serde(default, skip_serializing_if = "std::ops::Not::not")] pub sessions_storage_disabled: bool, + /// The server's sweep (`ai_sessions.rs`) deletes the backup of a session no push has + /// reached for this many days. The copies in members' browsers are untouched. Unset + /// keeps backups until the user deletes the session. Read from the workspace's own row + /// like `copilot_disabled`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub sessions_retention_days: Option, } /// Negotiated rates in USD per million tokens. An unset cache rate is read as the @@ -491,6 +497,9 @@ impl ModelPriceOverride { } } +/// Ten years: past any plausible retention, and well within what a day count is turned into. +pub const MAX_SESSIONS_RETENTION_DAYS: u32 = 3650; + impl AIConfig { pub fn validate_model_pricing(&self) -> Result<()> { for (key, price) in self.model_pricing.iter().flatten() { @@ -499,6 +508,17 @@ impl AIConfig { Ok(()) } + pub fn validate_sessions_retention(&self) -> Result<()> { + match self.sessions_retention_days { + Some(days) if !(1..=MAX_SESSIONS_RETENTION_DAYS).contains(&days) => { + Err(Error::BadRequest(format!( + "AI session retention must be between 1 and {MAX_SESSIONS_RETENTION_DAYS} days (got {days})" + ))) + } + _ => Ok(()), + } + } + pub fn has_providers(&self) -> bool { self.providers .as_ref() diff --git a/backend/windmill-api/src/ai_sessions.rs b/backend/windmill-api/src/ai_sessions.rs index 776b3a0561..65a8137c4e 100644 --- a/backend/windmill-api/src/ai_sessions.rs +++ b/backend/windmill-api/src/ai_sessions.rs @@ -30,8 +30,9 @@ use serde_json::value::RawValue; use std::sync::Arc; use windmill_api_auth::is_effectively_unscoped; use windmill_api_workspaces::ai_session_backups::{ - fallback_store, generation_prefix, storage_id, MAX_OBJECT_BYTES, + fallback_store, generation_prefix, primary_store, storage_id, MAX_OBJECT_BYTES, }; +use windmill_api_workspaces::workspaces::sessions_retention_days; use windmill_common::error::{Error, JsonResult, Result}; use windmill_common::utils::calculate_hash; use windmill_common::variables::{crypt_from_key_with_suffix, get_workspace_key}; @@ -65,6 +66,46 @@ const MAX_LISTED_OBJECTS: usize = 5000; const MAX_LIST_SCAN: usize = 50_000; const LIST_MAX: usize = 500; const IO_CONCURRENCY: usize = 8; +/// Sessions the retention sweep deletes per workspace and pass at most; the rest wait for +/// the next pass. +const SWEEP_MAX_PER_WORKSPACE: usize = 1000; +/// Session-level advisory lock of the retention sweep: one server at a time runs it. +const SWEEP_LOCK_ID: i64 = 0x5745_4550_4149; +/// The name of the sweep's record next to a session's markers (see `Backend::sweep_key`). +const SWEEP_RECORD: &str = "sweep"; +/// The name of a split push's token next to a session's markers (see `Backend::push_key`). +const PUSH_TOKEN: &str = "push"; + +/// A marker modified before this is past a retention of `days`. +fn retention_cutoff(days: u32) -> chrono::DateTime { + chrono::Utc::now() - chrono::Duration::days(i64::from(days)) +} + +/// What a key under the `index/` prefix is. +enum IndexEntry { + /// The marker that lists the session, named by its epoch. + Marker(u32), + /// The retention sweep's record (see `Backend::sweep_key`). + Sweep, + /// The token of a push split over parts (see `Backend::push_key`). + Push, +} + +/// The session a key under the `index/` prefix belongs to, and what the key is. +fn index_entry<'a>(index: &ObjectPath, key: &'a ObjectPath) -> Option<(&'a str, IndexEntry)> { + // `Path` drops the trailing delimiter, so the remainder starts with one. + let rel = key.as_ref().strip_prefix(index.as_ref())?; + let (sid, name) = rel.trim_start_matches('/').split_once('/')?; + if sid.is_empty() { + return None; + } + let entry = match name { + SWEEP_RECORD => IndexEntry::Sweep, + PUSH_TOKEN => IndexEntry::Push, + epoch => IndexEntry::Marker(epoch.parse().ok()?), + }; + Some((sid, entry)) +} pub fn workspaced_service() -> Router { Router::new() @@ -101,6 +142,9 @@ struct Backend { /// owed to the storage alone (a rotation deleted the older generation's copy anyway). storage_id: String, generation: i64, + /// `ai_config.sessions_retention_days`: a session whose marker is older is not listed, + /// whether or not the sweep has deleted it yet. + retention_days: Option, /// The store is the instance object store standing in for a workspace without storage /// of its own (`ai_session_backups::fallback_store`). fallback: bool, @@ -111,6 +155,12 @@ impl Backend { ObjectPath::from(format!("{}/index/", self.prefix)) } + /// The moment a marker's modification time must reach to count as live, under the + /// workspace's retention; `None` without one. + fn retention_cutoff(&self) -> Option> { + self.retention_days.map(retention_cutoff) + } + /// The marker that lists the session, named by the session's move count so that of a /// session two workspaces list, the copy moved last is told from the listing alone. fn index_key(&self, sid: &str, epoch: u32) -> ObjectPath { @@ -121,10 +171,18 @@ impl Backend { ObjectPath::from(format!("{}/index/{sid}/", self.prefix)) } - /// The token of the push split over parts in progress, under the session so a removal - /// or the next whole push clears it with the rest. + /// Written by the retention sweep before it deletes anything of a session, and deleted + /// last (`remove_session`): what finds a removal the sweep started and could not finish, + /// the markers being gone by then. Not an epoch, so nothing lists or pulls a session by it. + fn sweep_key(&self, sid: &str) -> ObjectPath { + ObjectPath::from(format!("{}/index/{sid}/{SWEEP_RECORD}", self.prefix)) + } + + /// The token of the push split over parts in progress, next to the markers so a removal + /// or the next whole push clears it with them, and the retention sweep, which walks the + /// markers, finds one a browser abandoned. fn push_key(&self, sid: &str) -> ObjectPath { - ObjectPath::from(format!("{}/sessions/{sid}/push", self.prefix)) + ObjectPath::from(format!("{}/index/{sid}/{PUSH_TOKEN}", self.prefix)) } fn session_prefix(&self, sid: &str) -> ObjectPath { @@ -310,7 +368,12 @@ impl Backend { let mut stream = self.store.list(Some(&prefix)); while let Some(meta) = stream.next().await { let meta = meta.map_err(object_store_error_to_error)?; - listed |= marker; + // The sweep's record is not a marker: a session it started removing is absent. + listed |= marker + && meta + .location + .filename() + .is_some_and(|name| name.parse::().is_ok()); acc = fold( acc, meta.location.as_ref(), @@ -415,17 +478,21 @@ fn require_json_object(kind: &str, raw: &RawValue, max_bytes: usize) -> Result<( /// no instance store to stand in, or the admin switched them off. Both read as /// `enabled: false` so the browser stops trying. async fn backend(authed: &ApiAuthed, db: &DB, w_id: &str) -> Result> { - let (disabled, generation, has_storage) = sqlx::query_as::<_, (Option, i64, bool)>( - "SELECT (ai_config->>'sessions_storage_disabled')::bool, ai_sessions_backup_generation, \ - large_file_storage IS NOT NULL FROM workspace_settings WHERE workspace_id = $1", - ) - .bind(w_id) - .fetch_optional(db) - .await? - .unwrap_or((None, 0, false)); + let (disabled, retention, generation, has_storage) = + sqlx::query_as::<_, (Option, Option, i64, bool)>( + "SELECT (ai_config->>'sessions_storage_disabled')::bool, \ + ai_config->'sessions_retention_days', ai_sessions_backup_generation, \ + large_file_storage IS NOT NULL \ + FROM workspace_settings WHERE workspace_id = $1", + ) + .bind(w_id) + .fetch_optional(db) + .await? + .unwrap_or((None, None, 0, false)); if disabled.unwrap_or(false) { return Ok(None); } + let retention_days = sessions_retention_days(retention.as_ref()); // Decided from the row the generation came from: the instance store is written only // under a generation read while the workspace had no storage of its own, which // configuring one moves past (`ai_session_backups`). @@ -459,6 +526,7 @@ async fn backend(authed: &ApiAuthed, db: &DB, w_id: &str) -> Result, @@ -511,6 +581,7 @@ async fn list( })); }; let prefix = backend.index_prefix(); + let cutoff = backend.retention_cutoff(); let mut stream = backend.store.list(Some(&prefix)); // One marker per session, whatever the session holds: the newest LIST_MAX are kept as // the scan goes (a min-heap drops the oldest), and the scan itself is bounded. @@ -526,19 +597,12 @@ async fn list( truncated = true; break; } - // `Path` drops the trailing delimiter, so the remainder starts with one. - let Some(rel) = meta.location.as_ref().strip_prefix(prefix.as_ref()) else { - continue; - }; - let Some((sid, epoch)) = rel.trim_start_matches('/').split_once('/') else { - continue; - }; - let Ok(epoch) = epoch.parse::() else { - continue; - }; - if sid.is_empty() || sid.contains('/') { + if cutoff.is_some_and(|cutoff| meta.last_modified < cutoff) { continue; } + let Some((sid, IndexEntry::Marker(epoch))) = index_entry(&prefix, &meta.location) else { + continue; + }; newest.push(std::cmp::Reverse(( meta.last_modified, epoch, @@ -1206,15 +1270,29 @@ async fn push_session(backend: &Backend, s: &PushedSession) -> Result<(usize, bo Ok((written, false)) } -/// The marker goes first so a removal cut short leaves nothing listed, then the head so +/// The markers go first so a removal cut short leaves nothing listed, then the head so /// nothing pulls either, and no push takes it for a session still there (see `push_session`). +/// The retention sweep's record goes last (see `Backend::sweep_key`). async fn remove_session(backend: &Backend, sid: &str) -> Result<()> { + let sweep = backend.sweep_key(sid); backend - .delete_prefix(&backend.index_session_prefix(sid)) + .store + .list(Some(&backend.index_session_prefix(sid))) + .map_err(object_store_error_to_error) + .try_for_each_concurrent(IO_CONCURRENCY, |meta| { + let sweep = &sweep; + async move { + if meta.location == *sweep { + return Ok(()); + } + backend.delete(&meta.location).await + } + }) .await?; backend.delete(&backend.head_key(sid)).await?; backend.delete_prefix(&backend.session_prefix(sid)).await?; - backend.delete_prefix(&backend.images_prefix(sid)).await + backend.delete_prefix(&backend.images_prefix(sid)).await?; + backend.delete(&sweep).await } /// One writer per session at a time, across servers: a push and a removal of the same @@ -1261,6 +1339,205 @@ async fn remove_session_locked(db: &DB, backend: &Backend, sid: &str) -> Result< result } +/// Deletes, in every workspace with `ai_config.sessions_retention_days`, the backups of the +/// sessions whose marker is older than that: the marker is rewritten by every push that +/// completes, so its modification time is the session's last activity as the storage clocks +/// it. For the monitor, on every server: a session-level advisory lock keeps one pass at a +/// time across them. The walk reads markers only, one object per session and nothing of what +/// the sessions hold, under each user's prefix in turn (`list_with_delimiter` names the +/// users), and deletes at most `SWEEP_MAX_PER_WORKSPACE` sessions per workspace and pass. A +/// session goes under its lock (`lock_session`), once its markers are listed again there and +/// still all older (see `sweep_session`). A removal cut short leaves the sweep's record next +/// to the markers, which the walk also collects, so the next pass finishes it. +pub async fn sweep_expired_ai_session_backups(db: &DB) { + let mut lock_conn = match db.acquire().await { + Ok(conn) => conn, + Err(e) => { + tracing::error!("AI session retention: could not acquire a connection: {e:#}"); + return; + } + }; + let locked: bool = match sqlx::query_scalar("SELECT pg_try_advisory_lock($1)") + .bind(SWEEP_LOCK_ID) + .fetch_one(&mut *lock_conn) + .await + { + Ok(locked) => locked, + Err(e) => { + tracing::error!("AI session retention: advisory lock failed: {e:#}"); + return; + } + }; + if !locked { + return; + } + if let Err(e) = sweep_workspaces(db).await { + tracing::error!("AI session retention sweep failed: {e:#}"); + } + if let Err(e) = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(SWEEP_LOCK_ID) + .execute(&mut *lock_conn) + .await + { + tracing::error!("AI session retention: advisory unlock failed: {e:#}"); + } +} + +async fn sweep_workspaces(db: &DB) -> Result<()> { + let workspaces = sqlx::query_as::<_, (String, Option, i64, bool)>( + "SELECT workspace_id, ai_config->'sessions_retention_days', ai_sessions_backup_generation, \ + large_file_storage IS NOT NULL \ + FROM workspace_settings \ + WHERE ai_config->'sessions_retention_days' IS NOT NULL", + ) + .fetch_all(db) + .await?; + for (w_id, retention, generation, has_storage) in workspaces { + let Some(days) = sessions_retention_days(retention.as_ref()) else { + continue; + }; + match sweep_workspace(db, &w_id, days, generation, has_storage).await { + Ok(0) => {} + Ok(deleted) => tracing::info!( + "AI session retention deleted {deleted} session backups of {w_id} older than {days} days" + ), + Err(e) => tracing::warn!("AI session retention sweep of {w_id}: {e:#}"), + } + } + Ok(()) +} + +/// `has_storage` comes from the row `generation` was read from, as in `backend`: the instance +/// store is swept only under a generation read while the workspace had no storage of its own. +async fn sweep_workspace( + db: &DB, + w_id: &str, + days: u32, + generation: i64, + has_storage: bool, +) -> Result { + let resolved = if has_storage { + primary_store(db, w_id).await? + } else { + fallback_store(db).await? + }; + let Some(resolved) = resolved else { + return Ok(0); + }; + let key = get_workspace_key(w_id, db).await?; + let (store, storage_id) = (resolved.store, resolved.storage_id); + let cutoff = retention_cutoff(days); + let root = ObjectPath::from(generation_prefix(w_id, generation)); + let users = store + .list_with_delimiter(Some(&root)) + .await + .map_err(object_store_error_to_error)? + .common_prefixes; + let mut deleted = 0; + for user_prefix in users { + let Some(user) = user_prefix.filename() else { + continue; + }; + // The sweep decrypts nothing; the cipher is only what a `Backend` is made of. + let backend = Backend { + store: store.clone(), + mc: crypt_from_key_with_suffix(&key, user), + prefix: user_prefix.to_string(), + storage_id: storage_id.clone(), + generation, + retention_days: Some(days), + fallback: resolved.fallback, + }; + let index = backend.index_prefix(); + let mut markers = backend.store.list(Some(&index)); + let mut expired = std::collections::BTreeSet::new(); + while let Some(meta) = markers.next().await { + let meta = meta.map_err(object_store_error_to_error)?; + let sid = match index_entry(&index, &meta.location) { + Some((sid, IndexEntry::Sweep)) => sid, + Some((sid, IndexEntry::Marker(_) | IndexEntry::Push)) + if meta.last_modified < cutoff => + { + sid + } + _ => continue, + }; + expired.insert(sid.to_string()); + if deleted + expired.len() >= SWEEP_MAX_PER_WORKSPACE { + break; + } + } + for sid in expired { + match sweep_session(db, &backend, &sid, cutoff).await { + Ok(true) => deleted += 1, + Ok(false) => {} + Err(e) => tracing::warn!( + "AI session retention left the backup of {sid} in {w_id} for the next pass: {e:#}" + ), + } + } + if deleted >= SWEEP_MAX_PER_WORKSPACE { + break; + } + } + Ok(deleted) +} + +/// True when the session was deleted. Under the session's lock its markers are listed again: +/// one a push renewed since the walk keeps the session. A session with none is left alone +/// while a push split over parts is between two of them (its token younger than the +/// retention) or it is gone, unless the sweep's record says a removal was started; an older +/// token is a split push a browser abandoned, whose landed parts nothing lists. The record is +/// written before anything is deleted and removed last, so a removal cut short is found again +/// by the next pass. +async fn sweep_session( + db: &DB, + backend: &Backend, + sid: &str, + cutoff: chrono::DateTime, +) -> Result { + let tx = lock_session(db, backend, sid).await?; + let result = async { + let (sweep, push) = (backend.sweep_key(sid), backend.push_key(sid)); + let mut entries = backend.store.list(Some(&backend.index_session_prefix(sid))); + let (mut listed, mut renewed, mut started, mut abandoned) = (false, false, false, false); + while let Some(meta) = entries.next().await { + let meta = meta.map_err(object_store_error_to_error)?; + if meta.location == sweep { + started = true; + } else if meta.location == push { + abandoned = meta.last_modified < cutoff; + } else { + listed = true; + renewed |= meta.last_modified >= cutoff; + } + } + if renewed { + // A push listed the session again over a removal cut short before its markers + // went, which had deleted nothing else. + if started { + backend.delete(&sweep).await?; + } + return Ok(false); + } + if !listed && !started && !abandoned { + return Ok(false); + } + if !started { + backend + .store + .put(&sweep, PutPayload::new()) + .await + .map_err(object_store_error_to_error)?; + } + remove_session(backend, sid).await?; + Ok(true) + } + .await; + tx.commit().await?; + result +} + async fn push( authed: ApiAuthed, Extension(db): Extension, diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index 715f0b6556..b70d80140a 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -71,6 +71,8 @@ mod ai_free_tier_ee; mod ai_free_tier_oss; #[cfg(feature = "parquet")] mod ai_sessions; +#[cfg(feature = "parquet")] +pub use ai_sessions::sweep_expired_ai_session_backups; mod ai_shared_artifacts; mod apps; mod apps_raw_bundle; diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index 55f244c143..a0a27eab4c 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -108,6 +108,7 @@ async fn edit_copilot_config( } ai_config.validate_model_pricing()?; + ai_config.validate_sessions_retention()?; let mut tx = db.begin().await?; @@ -148,6 +149,7 @@ async fn edit_copilot_config( let workspace_has_config = ai_config.has_providers(); let copilot_disabled = ai_config.copilot_disabled; let sessions_storage_disabled = ai_config.sessions_storage_disabled; + let sessions_retention_days = ai_config.sessions_retention_days; let instance_ai_config = sqlx::query_scalar!("SELECT value FROM global_settings WHERE name = 'ai_config'") .fetch_optional(&db) @@ -176,6 +178,7 @@ async fn edit_copilot_config( }; effective_ai_config.copilot_disabled = copilot_disabled; effective_ai_config.sessions_storage_disabled = sessions_storage_disabled; + effective_ai_config.sessions_retention_days = sessions_retention_days; Ok(Json(EditCopilotConfigResponse { effective_ai_config, @@ -217,6 +220,9 @@ async fn get_copilot_info( let sessions_storage_disabled = workspace_ai_config .as_ref() .is_some_and(|c| c.0.sessions_storage_disabled); + let sessions_retention_days = workspace_ai_config + .as_ref() + .and_then(|c| c.0.sessions_retention_days); let instance_config = sqlx::query_scalar!("SELECT value FROM global_settings WHERE name = 'ai_config'") .fetch_optional(&db) @@ -242,6 +248,7 @@ async fn get_copilot_info( }; effective.copilot_disabled = copilot_disabled; effective.sessions_storage_disabled = sessions_storage_disabled; + effective.sessions_retention_days = sessions_retention_days; Ok(Json(effective)) } diff --git a/docs/ai-session-backups.md b/docs/ai-session-backups.md index 3e0c747b8a..9af2bd3a84 100644 --- a/docs/ai-session-backups.md +++ b/docs/ai-session-backups.md @@ -201,7 +201,8 @@ its pulls are done, and just before they do the whole family is listed again (me backups were off included, since a move from another device can land in a workspace between the first listings and the pulls; a family of one, with nowhere else for a copy to show up, is not): a session a later copy of which showed up elsewhere is left, with the family, for the -next time. Only a user-initiated `deleteSession` removes the backup; the next push from +next time. Only a user-initiated `deleteSession`, and the retention sweep below, remove the +backup; the next push from another device that still has the session is refused with `needs_whole` (nothing of it is written), its row goes stale without a backoff, and that device's next flush sends the session whole; the workspace-lifecycle @@ -235,6 +236,37 @@ old one; a session whose own parts were answered from different storages is not all. The listing a restore starts with runs the same check, so a storage switch is noticed at the first push after it or on the next page load, whichever comes first. +## Retention + +`ai_config.sessions_retention_days` (per workspace, in the AI settings; unset by default; +the `sessions_storage_disabled` pattern: no migration, carried by settings export and the +CLI; 1 to 3650) puts an age on backups, counted from the last push of the session that +completed. It applies to the backup only: a browser keeps its copy whatever the retention, +and a backup swept while a browser still has the session comes back once that browser writes +to it again (its incremental push is refused and goes whole). + +The server sweeps the object store (`sweep_expired_ai_session_backups`, from the monitor about +every 40 minutes on each server, one pass at a time under a session-level advisory lock). For +every workspace with a retention it takes the store its backups live in, its own storage or +the instance store standing in, decided from the row it reads the generation from as the +routes do, names the users under the generation prefix (`list_with_delimiter`) and lists each +user's `index/` once: one object per session, nothing of what the sessions hold. A session +whose marker is older than the retention is removed under its lock (`lock_session`), once its +markers, listed again there, are still all older: a push that renewed the session between the +walk and the lock keeps it, and one split over parts either holds the lock or has the session +unlisted with its token next to the markers (`index/{sid}/push`), which the sweep leaves +alone while the token is younger than the retention: an older one is a push a browser +abandoned, whose landed parts nothing lists, and it goes the same way. Before deleting +anything the sweep writes a record next to the markers (`index/{sid}/sweep`, not an epoch, so +neither `list` nor `pull` counts it), and `remove_session` deletes it last: a removal cut +short, its markers already gone, is found by the next pass and finished, unless a push listed +the session again first. At most 1000 sessions per workspace and pass; the rest wait for the +next. `list` leaves an expired marker out of its answer meanwhile, so a browser never restores +a session the sweep has not reached. The marker's modification time is the storage's clock and +the cutoff the server's. The sweep reaches only the backups the routes would: a deleted +workspace's stay in its storage, and so do those a workspace keeps in the instance store once +`ai_sessions_instance_storage_fallback` is set to false. + ## Limits Push bodies are packed to about 8 MB (UTF-8 bytes as sent), at most 100 entries, 200 removals and diff --git a/frontend/src/lib/components/workspaceSettings/AISettings.svelte b/frontend/src/lib/components/workspaceSettings/AISettings.svelte index 5bc7974af2..a1dcc76758 100644 --- a/frontend/src/lib/components/workspaceSettings/AISettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/AISettings.svelte @@ -29,6 +29,7 @@ import AiUsagePanel from './AiUsagePanel.svelte' import { setCopilotInfo } from '$lib/aiStore' import { backupSettingsChanged } from '$lib/components/sessions/sessionMirror.svelte' + import TextInput from '../text_input/TextInput.svelte' import AIPromptsModal from '../settings/AIPromptsModal.svelte' import { Settings } from 'lucide-svelte' import { untrack } from 'svelte' @@ -81,6 +82,7 @@ let workspaceOverrideEditorOpened = $state(false) let copilotDisabled = $state(false) let sessionsStorageDisabled = $state(false) + let sessionsRetentionDays: number | undefined = $state(undefined) // --- Initial state for dirty tracking --- let initialAiProviders: Exclude = $state({}) @@ -93,6 +95,7 @@ let initialPrompts: Record = $state({}) let initialCopilotDisabled = $state(false) let initialSessionsStorageDisabled = $state(false) + let initialSessionsRetentionDays: number | undefined = $state(undefined) let lastLoadedConfigKey = $state(undefined) function clone(v: T): T { @@ -122,6 +125,7 @@ modelPricing = clone(config?.model_pricing ?? {}) copilotDisabled = config?.copilot_disabled === true sessionsStorageDisabled = config?.sessions_storage_disabled === true + sessionsRetentionDays = config?.sessions_retention_days for (const mode of ['edit', 'fix', 'gen']) { if (!(mode in customPrompts)) { customPrompts[mode] = '' @@ -140,6 +144,7 @@ initialPrompts = clone(customPrompts) initialCopilotDisabled = copilotDisabled initialSessionsStorageDisabled = sessionsStorageDisabled + initialSessionsRetentionDays = sessionsRetentionDays } export function loadFromConfig(config: AIConfig | undefined) { @@ -157,6 +162,7 @@ modelPricing = clone(initialModelPricing) copilotDisabled = initialCopilotDisabled sessionsStorageDisabled = initialSessionsStorageDisabled + sessionsRetentionDays = initialSessionsRetentionDays } $effect(() => { @@ -193,7 +199,8 @@ JSON.stringify(maxTokensPerModel) !== JSON.stringify(initialMaxTokensPerModel) || JSON.stringify(modelPricing) !== JSON.stringify(initialModelPricing) || copilotDisabled !== initialCopilotDisabled || - sessionsStorageDisabled !== initialSessionsStorageDisabled + sessionsStorageDisabled !== initialSessionsStorageDisabled || + sessionsRetentionDays !== initialSessionsRetentionDays ) $effect(() => { @@ -298,9 +305,11 @@ .filter(([_, prompt]) => prompt.trim().length > 0) .reduce((acc, [mode, prompt]) => ({ ...acc, [mode]: prompt }), {}) - // The flags are what a workspace on instance defaults still stores of its own. + // The flags and the retention are what a workspace on instance defaults still stores + // of its own. const copilot_disabled = copilotDisabled ? true : undefined const sessions_storage_disabled = sessionsStorageDisabled ? true : undefined + const sessions_retention_days = sessionsRetentionDays return Object.keys(aiProviders ?? {}).length > 0 ? { providers: aiProviders, @@ -312,16 +321,27 @@ Object.keys(maxTokensPerModel).length > 0 ? maxTokensPerModel : undefined, model_pricing: Object.keys(modelPricing).length > 0 ? modelPricing : undefined, copilot_disabled, - sessions_storage_disabled + sessions_storage_disabled, + sessions_retention_days } - : { copilot_disabled, sessions_storage_disabled } + : { copilot_disabled, sessions_storage_disabled, sessions_retention_days } } + // The server refuses anything outside this range; the input holds the same bounds. + const MAX_SESSIONS_RETENTION_DAYS = 3650 + let retentionInvalid = $derived( + sessionsRetentionDays !== undefined && + (!Number.isInteger(sessionsRetentionDays) || + sessionsRetentionDays < 1 || + sessionsRetentionDays > MAX_SESSIONS_RETENTION_DAYS) + ) + function isSaveDisabled(): boolean { return ( !Object.values(aiProviders).every((p) => p.resource_path) || (metadataModel != undefined && metadataModel.length === 0) || - (codeCompletionModel != undefined && codeCompletionModel.length === 0) + (codeCompletionModel != undefined && codeCompletionModel.length === 0) || + retentionInvalid ) } @@ -671,6 +691,33 @@ options={{ right: 'Do not back AI sessions up to the workspace storage' }} /> + +
    +
    + sessionsRetentionDays ?? '', + (v) => { + const n = typeof v === 'number' ? v : parseInt(v ?? '') + sessionsRetentionDays = Number.isNaN(n) ? undefined : n + } + } + /> +
    + days +
    +     {/if} From a48ae656ae59d600311f81ef357d08df5226515a Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 16 Sep 2026 09:57:43 +0200 Subject: [PATCH 138/144] feat: delete a browser's copy of an AI session past its workspace retention (#11156) * feat: delete a browser's copy of an AI session past its workspace retention Co-Authored-By: Claude Opus 5 (1M context) * fix: tell the AI session retention only to a member who can reach the workspace Co-Authored-By: Claude Opus 5 (1M context) * docs: keep the retention sweep's design narrative in the docs, not the code * fix: give the session retention its own route, leaving the status contract alone Co-Authored-By: Claude Opus 5 (1M context) * docs: shorten the retention route comment to its constraints * docs: name the two clocks in the retention setting, and the deploy window --------- Co-authored-by: Claude Opus 5 (1M context) --- ...c8a287bf1ff4b9d4301e3ea0efd8077936aff.json | 30 ++ backend/tests/session_workspace_status.rs | 68 ++++- .../windmill-api-workspaces/src/workspaces.rs | 40 +++ backend/windmill-api/openapi.yaml | 31 ++ docs/ai-session-backups.md | 104 +++++-- .../copilot/chat/HistoryManager.svelte.ts | 17 +- .../chat/artifacts/artifactsDB.test.ts | 2 +- .../copilot/chat/artifacts/artifactsDB.ts | 11 +- .../chat/files/attachedFilesDB.test.ts | 2 +- .../copilot/chat/files/attachedFilesDB.ts | 7 +- .../sessions/sessionMirror.svelte.ts | 12 +- .../components/sessions/sessionMirror.test.ts | 15 +- .../components/sessions/sessionMirrorPlan.ts | 6 +- .../sessions/sessionMirrorSignal.ts | 24 ++ .../sessions/sessionState.svelte.ts | 272 +++++++++++++++++- .../sessions/sessionStateIndexedDb.test.ts | 130 ++++++++- .../workspaceSettings/AISettings.svelte | 2 +- 17 files changed, 718 insertions(+), 55 deletions(-) create mode 100644 backend/.sqlx/query-8eee14066c86b4a4ef921576277c8a287bf1ff4b9d4301e3ea0efd8077936aff.json diff --git a/backend/.sqlx/query-8eee14066c86b4a4ef921576277c8a287bf1ff4b9d4301e3ea0efd8077936aff.json b/backend/.sqlx/query-8eee14066c86b4a4ef921576277c8a287bf1ff4b9d4301e3ea0efd8077936aff.json new file mode 100644 index 0000000000..552a38a0d4 --- /dev/null +++ b/backend/.sqlx/query-8eee14066c86b4a4ef921576277c8a287bf1ff4b9d4301e3ea0efd8077936aff.json @@ -0,0 +1,30 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT workspace_settings.workspace_id AS \"id!\",\n workspace_settings.ai_config->'sessions_retention_days' AS retention\n FROM workspace_settings\n LEFT JOIN usr ON usr.workspace_id = workspace_settings.workspace_id AND usr.email = $2\n WHERE workspace_settings.workspace_id = ANY($1)\n AND ($3 OR (usr.email IS NOT NULL AND NOT usr.disabled))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "retention", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "TextArray", + "Text", + "Bool" + ] + }, + "nullable": [ + false, + null + ] + }, + "hash": "8eee14066c86b4a4ef921576277c8a287bf1ff4b9d4301e3ea0efd8077936aff" +} diff --git a/backend/tests/session_workspace_status.rs b/backend/tests/session_workspace_status.rs index b4fc98da5e..2ed4ac503a 100644 --- a/backend/tests/session_workspace_status.rs +++ b/backend/tests/session_workspace_status.rs @@ -3,18 +3,23 @@ //! extractor actually grants. Membership is not the only path: a superadmin is authed into //! any existing workspace without a `usr` row, and `admins` has no `usr` rows at all, so //! answering from `usr` alone reports live workspaces as unresolvable and the client deletes -//! sessions that still work. +//! sessions that still work. `POST /workspaces/session_workspace_retention`, the AI session +//! retention the same client deletes its own copies by, is a workspace setting and answers to +//! the stricter bar, which is why the two are separate routes and tested together. use serde_json::json; use sqlx::{Pool, Postgres}; use std::collections::HashMap; use windmill_test_utils::*; -async fn status(port: u16, token: &str, ids: &[&str]) -> anyhow::Result> { +async fn post( + port: u16, + route: &str, + token: &str, + ids: &[&str], +) -> anyhow::Result { let resp = reqwest::Client::new() - .post(format!( - "http://localhost:{port}/api/workspaces/session_workspace_status" - )) + .post(format!("http://localhost:{port}/api/workspaces/{route}")) .header("Authorization", format!("Bearer {token}")) .json(&json!({ "workspace_ids": ids })) .send() @@ -23,6 +28,14 @@ async fn status(port: u16, token: &str, ids: &[&str]) -> anyhow::Result anyhow::Result> { + post(port, "session_workspace_status", token, ids).await +} + +async fn retention(port: u16, token: &str, ids: &[&str]) -> anyhow::Result> { + post(port, "session_workspace_retention", token, ids).await +} + #[sqlx::test(fixtures("base", "session_workspace_status"))] async fn test_superadmin_reaches_workspaces_without_a_usr_row( db: Pool, @@ -60,3 +73,48 @@ async fn test_superadmin_reaches_workspaces_without_a_usr_row( Ok(()) } + +/// The retention a browser deletes its own copies by is a workspace setting, so unlike the +/// status it is told only to a caller the authed extractor would let in. +#[sqlx::test(fixtures("base", "session_workspace_status"))] +async fn test_session_retention_is_told_only_to_members_who_can_be_authed( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let ids = ["foreign-workspace", "test-workspace", "no-such-workspace"]; + sqlx::query( + "UPDATE workspace_settings SET ai_config = '{\"sessions_retention_days\": 7}' \ + WHERE workspace_id IN ('test-workspace', 'foreign-workspace')", + ) + .execute(&db) + .await?; + + // test@windmill.dev is a superadmin: authed into every workspace that exists. + let sa = retention(port, "SECRET_TOKEN", &ids).await?; + assert_eq!(sa["test-workspace"], 7); + assert_eq!(sa["foreign-workspace"], 7); + assert!(!sa.contains_key("no-such-workspace")); + + // test2@windmill.dev is a member of test-workspace only. + let usr = retention(port, "SECRET_TOKEN_2", &ids).await?; + assert_eq!(usr["test-workspace"], 7); + assert!(!usr.contains_key("foreign-workspace")); + + // A disabled membership still reconciles its sessions — the status stays `active` — but + // cannot be authed into the workspace, so it is told no setting. + sqlx::query("UPDATE usr SET disabled = true WHERE workspace_id = 'test-workspace'") + .execute(&db) + .await?; + assert_eq!( + status(port, "SECRET_TOKEN_2", &ids).await?["test-workspace"], + "active" + ); + assert!(!retention(port, "SECRET_TOKEN_2", &ids) + .await? + .contains_key("test-workspace")); + + Ok(()) +} diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index c29d99d05b..3330ed697f 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -227,6 +227,10 @@ pub fn global_service() -> Router { .route("/list", get(list_workspaces)) .route("/users", get(user_workspaces)) .route("/session_workspace_status", post(session_workspace_status)) + .route( + "/session_workspace_retention", + post(session_workspace_retention), + ) .route("/create", post(create_workspace)) .route("/create_fork", post(deprecated_create_workspace_fork)) .route("/exists", post(exists_workspace)) @@ -5686,6 +5690,42 @@ async fn session_workspace_status( Ok(Json(statuses)) } +/// The AI session retention a browser deletes its local copies by (docs/ai-session-backups.md). +/// Its own route, not a field on the status above, whose shape an older tab still reads. Unlike +/// a status, it answers only for a workspace this caller can be authed into: a setting is the +/// workspace's to tell, so a disabled membership gets none though its sessions still reconcile. +async fn session_workspace_retention( + Extension(db): Extension, + authed: ApiAuthed, + Json(req): Json, +) -> JsonResult> { + if req.workspace_ids.len() > 1000 { + return Err(Error::BadRequest( + "Too many workspace ids (max 1000)".to_string(), + )); + } + let email = &authed.email; + let is_superadmin = windmill_api_auth::is_super_admin_authed(&db, &authed).await?; + let rows = sqlx::query!( + "SELECT workspace_settings.workspace_id AS \"id!\", + workspace_settings.ai_config->'sessions_retention_days' AS retention + FROM workspace_settings + LEFT JOIN usr ON usr.workspace_id = workspace_settings.workspace_id AND usr.email = $2 + WHERE workspace_settings.workspace_id = ANY($1) + AND ($3 OR (usr.email IS NOT NULL AND NOT usr.disabled))", + &req.workspace_ids[..], + email, + is_superadmin, + ) + .fetch_all(&db) + .await?; + let days = rows + .into_iter() + .filter_map(|r| sessions_retention_days(r.retention.as_ref()).map(|days| (r.id, days))) + .collect(); + Ok(Json(days)) +} + /// The instance critical alert channels belong to the instance operator, who on cloud is /// not the workspace owner and never opted into a tenant's job failures. Fork workspaces run /// throwaway copies of their parent's runnables, so instance-wide operational alerting must diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 9ce1aef272..c88a298646 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1277,6 +1277,37 @@ paths: - archived - deleted + /workspaces/session_workspace_retention: + post: + summary: get the AI session retention of workspaces referenced by client-side sessions + operationId: getSessionWorkspaceRetention + tags: + - workspace + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + workspace_ids: + type: array + items: + type: string + required: + - workspace_ids + responses: + "200": + description: >- + map of workspace id to its `ai_config.sessions_retention_days`; a workspace + without a retention, or one the caller cannot be authenticated into, is absent + content: + application/json: + schema: + type: object + additionalProperties: + type: integer + /w/{workspace}/workspaces/get_as_superadmin: get: summary: get workspace as super admin (require to be super admin) diff --git a/docs/ai-session-backups.md b/docs/ai-session-backups.md index 9af2bd3a84..ffec41e182 100644 --- a/docs/ai-session-backups.md +++ b/docs/ai-session-backups.md @@ -240,32 +240,86 @@ the first push after it or on the next page load, whichever comes first. `ai_config.sessions_retention_days` (per workspace, in the AI settings; unset by default; the `sessions_storage_disabled` pattern: no migration, carried by settings export and the -CLI; 1 to 3650) puts an age on backups, counted from the last push of the session that -completed. It applies to the backup only: a browser keeps its copy whatever the retention, -and a backup swept while a browser still has the session comes back once that browser writes -to it again (its incremental push is refused and goes whole). +CLI; 1 to 3650) puts an age on sessions, counted from their last activity. Each side applies +it with its own clock against its own timestamps, so no clock is compared with another +machine's, and the two do not time the same event: the server counts the last push that +completed, a browser its last local activity, which includes reading new messages and is not +pushed. A backup swept while a browser still reads its copy comes back once that browser +writes to the session again (its incremental push is refused and goes whole): -The server sweeps the object store (`sweep_expired_ai_session_backups`, from the monitor about -every 40 minutes on each server, one pass at a time under a session-level advisory lock). For -every workspace with a retention it takes the store its backups live in, its own storage or -the instance store standing in, decided from the row it reads the generation from as the -routes do, names the users under the generation prefix (`list_with_delimiter`) and lists each -user's `index/` once: one object per session, nothing of what the sessions hold. A session -whose marker is older than the retention is removed under its lock (`lock_session`), once its -markers, listed again there, are still all older: a push that renewed the session between the -walk and the lock keeps it, and one split over parts either holds the lock or has the session -unlisted with its token next to the markers (`index/{sid}/push`), which the sweep leaves -alone while the token is younger than the retention: an older one is a push a browser -abandoned, whose landed parts nothing lists, and it goes the same way. Before deleting -anything the sweep writes a record next to the markers (`index/{sid}/sweep`, not an epoch, so -neither `list` nor `pull` counts it), and `remove_session` deletes it last: a removal cut -short, its markers already gone, is found by the next pass and finished, unless a push listed -the session again first. At most 1000 sessions per workspace and pass; the rest wait for the -next. `list` leaves an expired marker out of its answer meanwhile, so a browser never restores -a session the sweep has not reached. The marker's modification time is the storage's clock and -the cutoff the server's. The sweep reaches only the backups the routes would: a deleted -workspace's stay in its storage, and so do those a workspace keeps in the instance store once -`ai_sessions_instance_storage_fallback` is set to false. +- The server sweeps the object store (`sweep_expired_ai_session_backups`, from the monitor + about every 40 minutes on each server, one pass at a time under a session-level advisory + lock). For every workspace with a retention it takes the store its backups live in, its + own storage or the instance store standing in, decided from the row it reads the + generation from as the routes do, names the users under the generation prefix + (`list_with_delimiter`) and lists each user's `index/` once: one object + per session, nothing of what the sessions hold. A session whose marker is older than the + retention is removed under its lock (`lock_session`), once its markers, listed again + there, are still all older: a push that renewed the session between the walk and the lock + keeps it, and one split over parts either holds the lock or has the session unlisted with + its token next to the markers (`index/{sid}/push`), which the sweep leaves alone while the + token is younger than the retention: an older one is a push a browser abandoned, whose + landed parts nothing lists, and it goes the same way. Before deleting anything the sweep + writes a record next to + the markers (`index/{sid}/sweep`, not an epoch, so neither `list` nor `pull` counts it), + and `remove_session` deletes it last: a removal cut short, its markers already gone, is + found by the next pass and finished, unless a push listed the session again first. At + most 1000 sessions per workspace and pass; the rest wait for the next. `list` leaves an + expired marker out of its answer meanwhile, so a browser never restores a session the + sweep has not reached. The marker's modification time is the storage's clock and the + cutoff the server's. The sweep reaches only the backups the routes would: a deleted + workspace's stay in its storage, and so do those a workspace keeps in the instance store + once `ai_sessions_instance_storage_fallback` is set to false. +- The browser sweeps its own stores when a tab resolves the logged-in user + (`sweepExpiredSessions`, from the one `onUserChange` in `sessionState.svelte.ts`), before + that tab reads a single session. A session whose last activity is older than the retention + by the browser's clock is deleted locally, record, chats, images, attached files and + artifacts. A restored session carries the backup's time as its last activity, the storage's + clock, so it counts from the later of that and the moment it was restored here + (`restoredAt`): a browser clock ahead of the storage's never deletes a session it just + brought back. Archived sessions count like any other, and persisted unsent drafts by their + pending workspace. + + The stores are shared by the user's tabs, and each keeps copies of the sessions in memory, + so every tab holds a shared Web Lock from before it reads them until it stops using them, + and the sweep deletes only while holding that lock exclusively, requested if available: + granted exactly when no tab of the user has the sessions loaded, which is why the sweep + runs where it does and nowhere else. Nothing holds a copy of what it deletes and nothing + writes the stores meanwhile, so it deletes one record at a time and without re-reading. It + also takes the tab lock the flush and the restore take, again only if available, so neither + plans nor stages a session half deleted; like the restore, it does not run where Web Locks + do not exist. With several tabs open nothing is swept, until one of them reloads alone. + + The hold is only as good as the tabs that take it, so a tab still running a build from before + it has the sessions loaded and holds nothing. A tab loaded after that one, across a deploy, + can sweep a session the older tab has in memory, and a write there afterwards brings the + record back without its chats, which the next flush pushes. It needs a tab left open across a + deploy, a session untouched for the whole retention, and the user going back to that session + in the older tab; the next sweep deletes it again. The same window is open to the + workspace-lifecycle delete in `reconcileSessionsLifecycle`, which no lock guards at all. + + What deletes is the retention the server gives as the sweep runs, asked for under both locks + (`POST /workspaces/session_workspace_retention`, its own route rather than a field on the + lifecycle status, whose answer a tab loaded before this version still reads). Never a + remembered one: a retention raised or cleared since would otherwise delete a session that is + now within it, and a persisted unsent draft has no backup to come back from. What the sweep + keeps in localStorage decides only whether to ask again — it asks when it has asked nothing + yet, when the answer it has is a day old, or when that answer marks a session expired — so + an ordinary load costs no request at all. An answer that does not arrive within five seconds + leaves the sessions for the next load rather than delete on what this browser guessed. That + route answers for a workspace the caller can be authed into, unlike the status: a status is + what to do with the caller's own sessions, a setting is the workspace's to tell, so a + disabled membership is told nothing though its sessions still reconcile. + + Each session's record goes before its pieces, so nothing plans a push for it afterwards, + and a localStorage key written before the record and removed once every piece is gone makes + a later sweep finish a deletion that failed, unless a restore brought the session back + since. The record is deleted without the tombstone a user delete leaves, which is what lets + a restore bring it back. The session's dirty mark and sync row go with it (`sessionSwept`), + unless the row still carries a removal or a restore's staging. Nothing is sent to the storage: the local + copy's age says nothing about another device's, which may have pushed the session since, + and the server applies the rule to the backup on its own. A session swept here that the + storage still lists comes back on the next restore. ## Limits diff --git a/frontend/src/lib/components/copilot/chat/HistoryManager.svelte.ts b/frontend/src/lib/components/copilot/chat/HistoryManager.svelte.ts index 4edac6dc0a..d956b4c0c0 100644 --- a/frontend/src/lib/components/copilot/chat/HistoryManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/HistoryManager.svelte.ts @@ -267,15 +267,22 @@ export async function importStoredChats( return true } -/** Deletes these chats of the session (with their images) and these images: what an earlier - * restore staged for it and the backup no longer has. False when nothing could be deleted. */ +/** Every chat tagged with the session, and their images: for a session past its workspace's + * retention, which no runtime has mounted. */ +export function deleteSessionChats(sessionId: string, email: string): Promise { + return pruneSessionChats(sessionId, undefined, new Set(), email) +} + +/** Deletes chats of the session (with their images) and these images: what an earlier restore + * staged for it and the backup no longer has. `chats` names the ones to go; undefined is every + * chat of the session. False when nothing could be deleted. */ export async function pruneSessionChats( sessionId: string, - chats: Set, + chats: Set | undefined, images: Set, email: string ): Promise { - if (chats.size === 0 && images.size === 0) return true + if (chats?.size === 0 && images.size === 0) return true const db = await backupDb(email) if (!db) return false try { @@ -283,7 +290,7 @@ export async function pruneSessionChats( const chatStore = tx.objectStore('chats') const imageStore = tx.objectStore('images') for (const chatId of await chatStore.index('by-session').getAllKeys(sessionId)) { - if (!chats.has(String(chatId))) continue + if (chats && !chats.has(String(chatId))) continue await chatStore.delete(chatId) const keys = await imageStore .index('by-chat') diff --git a/frontend/src/lib/components/copilot/chat/artifacts/artifactsDB.test.ts b/frontend/src/lib/components/copilot/chat/artifacts/artifactsDB.test.ts index d70566ad18..a0fb5a4a4b 100644 --- a/frontend/src/lib/components/copilot/chat/artifacts/artifactsDB.test.ts +++ b/frontend/src/lib/components/copilot/chat/artifacts/artifactsDB.test.ts @@ -220,7 +220,7 @@ describe('artifactsDB', () => { expect(await noDb.getArtifact('a1')).toBeUndefined() expect(await noDb.listArtifactsForSession('s1')).toEqual([]) await expect(noDb.deleteArtifact('a1')).resolves.toBeUndefined() - await expect(noDb.deleteArtifactsForSession('s1')).resolves.toBeUndefined() + await expect(noDb.deleteArtifactsForSession('s1')).resolves.toBe(false) }) it('rejects a version read it could not make, instead of reading as absent', async () => { diff --git a/frontend/src/lib/components/copilot/chat/artifacts/artifactsDB.ts b/frontend/src/lib/components/copilot/chat/artifacts/artifactsDB.ts index e7ff4b1f60..f0cb32d71f 100644 --- a/frontend/src/lib/components/copilot/chat/artifacts/artifactsDB.ts +++ b/frontend/src/lib/components/copilot/chat/artifacts/artifactsDB.ts @@ -449,9 +449,14 @@ export async function pruneSessionArtifacts( } } -export async function deleteArtifactsForSession(sessionId: string): Promise { +/** False when the store could not be reached or the deletion failed. With `email`, only that + * user's store is touched: a caller that captured its user must not follow an account switch. */ +export async function deleteArtifactsForSession( + sessionId: string, + email?: string +): Promise { const db = await getDB() - if (!db) return + if (!db || (email !== undefined && db.name !== scopedKeyFor(ARTIFACTS_DB, email))) return false try { const tx = db.transaction(['items', 'versions'], 'readwrite') const items = tx.objectStore('items') @@ -464,8 +469,10 @@ export async function deleteArtifactsForSession(sessionId: string): Promise { putItem({ id: 'a', sessionId: 's1', kind: 'snapshot', name: 'x.txt', addedAt: 0 }) ).resolves.toBeUndefined() await expect(deleteItem('a')).resolves.toBeUndefined() - await expect(deleteItemsForSession('s1')).resolves.toBeUndefined() + await deleteItemsForSession('s1') }) it('does not throw when requesting persistent storage', async () => { diff --git a/frontend/src/lib/components/copilot/chat/files/attachedFilesDB.ts b/frontend/src/lib/components/copilot/chat/files/attachedFilesDB.ts index fe051ef8cc..dc872845cd 100644 --- a/frontend/src/lib/components/copilot/chat/files/attachedFilesDB.ts +++ b/frontend/src/lib/components/copilot/chat/files/attachedFilesDB.ts @@ -91,9 +91,10 @@ export async function deleteItem(id: string): Promise { await db?.delete('items', id) } -export async function deleteItemsForSession(sessionId: string): Promise { +/** False when the store could not be reached or the deletion failed. */ +export async function deleteItemsForSession(sessionId: string): Promise { const db = await getDB() - if (!db) return + if (!db) return false try { const tx = db.transaction('items', 'readwrite') const index = tx.store.index('by-session') @@ -103,8 +104,10 @@ export async function deleteItemsForSession(sessionId: string): Promise { cursor = await cursor.continue() } await tx.done + return true } catch (err) { console.error('Could not delete attached files for session', err) + return false } } diff --git a/frontend/src/lib/components/sessions/sessionMirror.svelte.ts b/frontend/src/lib/components/sessions/sessionMirror.svelte.ts index a18e2febd6..bfa1621822 100644 --- a/frontend/src/lib/components/sessions/sessionMirror.svelte.ts +++ b/frontend/src/lib/components/sessions/sessionMirror.svelte.ts @@ -27,7 +27,7 @@ import { getCurrentUserEmail, onUserChange, scopedKey, scopedKeyFor } from '$lib import { logFeatureUsage } from '$lib/utils/featureUsage' import { randomUUID } from '$lib/utils/uuid' import { workspaceRootId } from './sessionScope.svelte' -import { onMirrorSignal } from './sessionMirrorSignal' +import { onMirrorSignal, onSessionSwept, sessionsLockName } from './sessionMirrorSignal' import { importSessions, isSessionTombstoned, @@ -385,7 +385,7 @@ function hasWebLocks(): boolean { async function withUserLock(email: string, fn: () => Promise, wait = false): Promise { const locks = webLocks() if (!locks) return fn() - await locks.request(`wm-ai-sessions-mirror::${email}`, { ifAvailable: !wait }, async (lock) => { + await locks.request(sessionsLockName(email), { ifAvailable: !wait }, async (lock) => { if (lock) await fn() // The other tab's flush read the marks before this one's were written: try again // once it is done, rather than wait for the next write or load. @@ -1572,6 +1572,14 @@ export function backupSettingsChanged(ws: string): void { // --- Wiring --- if (BROWSER) { + // Nothing pushes a swept session again, so its mark and sync row are dead weight; a row + // still carrying a removal or a restore's staging is left to those. + onSessionSwept(async (id, email) => { + if (email !== getCurrentUserEmail()) return + dropDirty(id) + const row = await readSync(id, email) + if (row && !row.removed && !row.staging) await deleteSync([id], email) + }) onMirrorSignal((signal) => { // A mark for another user waits for that user's next load. const mine = !signal.email || signal.email === getCurrentUserEmail() diff --git a/frontend/src/lib/components/sessions/sessionMirror.test.ts b/frontend/src/lib/components/sessions/sessionMirror.test.ts index ff2c8373da..7676662d31 100644 --- a/frontend/src/lib/components/sessions/sessionMirror.test.ts +++ b/frontend/src/lib/components/sessions/sessionMirror.test.ts @@ -93,7 +93,7 @@ import { sessionState, type Session } from './sessionState.svelte' -import { markSessionDirty } from './sessionMirrorSignal' +import { markSessionDirty, sessionSwept } from './sessionMirrorSignal' import { __flushForTesting, __resetMirrorForTesting, @@ -268,6 +268,19 @@ describe('sessionMirror flush', () => { await __settleForTesting() }) + it('forgets the sync row of a session the retention swept, unless it carries a removal', async () => { + await __writeSyncForTesting( + [ + { id: 'swept', ws: 'admins', head: '', chats: {}, images: {} }, + { id: 'swept-removed', ws: 'admins', head: '', chats: {}, images: {}, removed: true } + ], + EMAIL + ) + await sessionSwept('swept', EMAIL) + await sessionSwept('swept-removed', EMAIL) + expect((await __syncRowsForTesting(EMAIL)).map((r) => r.id)).toEqual(['swept-removed']) + }) + it('keeps a delete filed on the sync row while the first push is still in flight', async () => { const s: Session = { id: 'sr', name: 'session-1', createdAt: 1, workspace_id: 'ws' } sessionState.sessions = [s] diff --git a/frontend/src/lib/components/sessions/sessionMirrorPlan.ts b/frontend/src/lib/components/sessions/sessionMirrorPlan.ts index a664e61f5e..caea48c792 100644 --- a/frontend/src/lib/components/sessions/sessionMirrorPlan.ts +++ b/frontend/src/lib/components/sessions/sessionMirrorPlan.ts @@ -79,9 +79,9 @@ export function isFallbackStorage(name: string): boolean { /** * The part of a session record the backup keeps. Left out on purpose: `name` (a * per-browser counter), the unsent-draft fields (`pending_*`, `draftPrompt`, - * `autoSendDraftAt`), `workspace_root_id` (derived on import), `transient`, and the two - * fields reading a session bumps (`lastSeenCount`, `lastActivityAt`) — so opening a - * session and reading its new messages never costs a push. + * `autoSendDraftAt`), `workspace_root_id` (derived on import), `transient`, `restoredAt` + * (this browser's clock), and the two fields reading a session bumps (`lastSeenCount`, + * `lastActivityAt`) — so opening a session and reading its new messages never costs a push. */ export type SessionHead = Pick< Session, diff --git a/frontend/src/lib/components/sessions/sessionMirrorSignal.ts b/frontend/src/lib/components/sessions/sessionMirrorSignal.ts index eb936d2e22..abe600b4a6 100644 --- a/frontend/src/lib/components/sessions/sessionMirrorSignal.ts +++ b/frontend/src/lib/components/sessions/sessionMirrorSignal.ts @@ -29,6 +29,13 @@ export function markSessionRemoved(sessionId: string, workspaceId?: string, emai emit({ kind: 'removed', sessionId, workspaceId, email }) } +/** The Web Lock one tab of the user holds while it reads or writes the stores wholesale: the + * backup's flush and restore, and the retention sweep, which must not interleave with either + * (a flush planning a session half deleted would push the deletions to the backup). */ +export function sessionsLockName(email: string): string { + return `wm-ai-sessions-mirror::${email}` +} + export function onMirrorSignal(fn: (signal: MirrorSignal) => void): void { handler = fn const replay = buffered @@ -36,7 +43,24 @@ export function onMirrorSignal(fn: (signal: MirrorSignal) => void): void { for (const signal of replay) fn(signal) } +let sweptHandler: ((sessionId: string, email: string) => Promise) | undefined + +/** The retention sweep deleted this session's local copy in the store of `email`: what the + * backup keeps of it in this browser goes too. Awaited under the sweep's tab lock. */ +export async function sessionSwept(sessionId: string, email: string): Promise { + try { + await sweptHandler?.(sessionId, email) + } catch (e) { + console.error('Could not forget the backup state of a swept session', e) + } +} + +export function onSessionSwept(fn: (sessionId: string, email: string) => Promise): void { + sweptHandler = fn +} + export function __resetMirrorSignalForTesting(): void { handler = undefined + sweptHandler = undefined buffered = [] } diff --git a/frontend/src/lib/components/sessions/sessionState.svelte.ts b/frontend/src/lib/components/sessions/sessionState.svelte.ts index 2670b3fa4d..3014e6d5e7 100644 --- a/frontend/src/lib/components/sessions/sessionState.svelte.ts +++ b/frontend/src/lib/components/sessions/sessionState.svelte.ts @@ -27,7 +27,13 @@ import { userScopedDb } from '$lib/userScopedDb' import { emailOfScopedKey, scopedKeyFor } from '$lib/userScopedStorage' import { deleteItemsForSession } from '../copilot/chat/files/attachedFilesDB' import { deleteArtifactsForSession } from '../copilot/chat/artifacts/artifactsDB' -import { markSessionDirty, markSessionRemoved } from './sessionMirrorSignal' +import { deleteSessionChats } from '../copilot/chat/HistoryManager.svelte' +import { + markSessionDirty, + markSessionRemoved, + sessionSwept, + sessionsLockName +} from './sessionMirrorSignal' // Switch the global workspace iff the target differs from the active one // and is non-empty. Centralises the "session needs its workspace in focus" @@ -124,6 +130,11 @@ export type Session = { // Absent on records last written before the field existed; readers fall back // to createdAt via sessionLastActivityAt. lastActivityAt?: number + // When this browser restored the session from its backup, by this browser's clock. + // The restore sets `lastActivityAt` to the backup's time, the storage's clock; the + // retention counts from whichever is later, so a browser clock ahead of the storage's + // never deletes a session it just brought back. Not backed up. + restoredAt?: number // Per-session unread watermark: the displayMessages count the last time // the user was on this session's page. Compared against the runtime's // current message count to derive the unread badge (see sessionUnread). @@ -438,8 +449,15 @@ export function __resetDeletedSessionIdsForTesting(): void { // The one way to remove a session's record. Tombstones BEFORE awaiting the delete so a // putSession racing this transaction cannot commit its write behind it — a direct // db.delete elsewhere would silently reopen that window. -async function deleteSessionRow(db: IDBPDatabase, id: string): Promise { - deletedSessionIds.add(id) +async function deleteSessionRow( + db: IDBPDatabase, + id: string, + // The retention sweep passes false. It holds the in-use lock exclusively, so no write can + // race its delete, and the backup may still hold the session: a tombstone would refuse the + // restore that is meant to bring it back. + tombstone = true +): Promise { + if (tombstone) deletedSessionIds.add(id) await db.delete('sessions', id) } @@ -634,6 +652,242 @@ export async function reconcileSessionsLifecycle(): Promise { } } +// --- Retention --- + +const DAY_MS = 24 * 60 * 60 * 1000 + +// Past the retention by this browser's clock, counted from the later of the session's last +// activity and its restore here: a restored session carries the backup's time, the storage's +// clock, so without `restoredAt` a browser running ahead would delete what it just brought +// back. Archived sessions count like any other. +function isSessionExpired( + session: Session, + retentionDays: number | undefined, + now: number +): boolean { + if (retentionDays === undefined || !(retentionDays >= 1)) return false + const since = Math.max(sessionLastActivityAt(session), session.restoredAt ?? 0) + return since < now - retentionDays * DAY_MS +} + +// What the server last told this browser, and when. It decides whether the sweep asks again, +// and nothing else: a retention raised or cleared since must not delete a session, and a +// persisted unsent draft has no backup to come back from. +const RETENTION_DAYS = 'windmill_sessions_retention_days' + +// Nothing remembered for longer than this is trusted even to say there is nothing to ask +// about, so a retention lowered while this browser saw nothing expiring still takes effect. +const RETENTION_STALE_MS = 24 * 60 * 60 * 1000 + +interface RememberedRetention { + at: number + days: Record +} + +function rememberRetention(email: string, days: Record): void { + try { + const remembered: RememberedRetention = { at: Date.now(), days } + localStorage.setItem(scopedKeyFor(RETENTION_DAYS, email), JSON.stringify(remembered)) + } catch {} +} + +function rememberedRetention(email: string): RememberedRetention | undefined { + try { + const stored = localStorage.getItem(scopedKeyFor(RETENTION_DAYS, email)) + const remembered = stored ? JSON.parse(stored) : undefined + if (remembered?.days && typeof remembered.days === 'object') { + return remembered as RememberedRetention + } + } catch {} + return undefined +} + +// How long the sweep waits for the retention of the workspaces it is about to sweep in. The +// tab reads its sessions after the sweep, so a request nothing answers costs the list this +// much and no more, and only in a tab that had something to delete. +const RETENTION_ASK_MS = 5000 + +// The retention the server gives now, or undefined when this browser could not be told: a +// session is deleted only on an answer of the moment. +async function askRetention(workspaceIds: string[]): Promise | undefined> { + try { + return await Promise.race([ + WorkspaceService.getSessionWorkspaceRetention({ + requestBody: { workspace_ids: workspaceIds } + }), + new Promise((resolve) => setTimeout(() => resolve(undefined), RETENTION_ASK_MS)) + ]) + } catch (e) { + console.error('Failed to read the AI session retention of the workspaces', e) + return undefined + } +} + +// One key per session this browser swept whose pieces are not all deleted yet. +const RETENTION_PENDING = 'windmill_sessions_retention_pending' + +function retentionPendingPrefix(email: string): string { + return `${scopedKeyFor(RETENTION_PENDING, email)}::` +} + +function forgetRetentionPending(email: string, id: string): void { + try { + localStorage.removeItem(retentionPendingPrefix(email) + id) + } catch {} +} + +function retentionPending(email: string): string[] { + const prefix = retentionPendingPrefix(email) + const ids: string[] = [] + try { + for (let i = 0; i < localStorage.length; i++) { + const key = localStorage.key(i) + if (key?.startsWith(prefix)) ids.push(key.slice(prefix.length)) + } + } catch {} + return ids +} + +function webLocks(): LockManager | undefined { + return typeof navigator === 'undefined' ? undefined : (navigator as { locks?: LockManager }).locks +} + +// Held, shared, by every tab from before it reads the user's sessions until it stops using +// them: the stores are shared, and each tab keeps copies of the sessions in memory, so the +// sweep deletes only while holding this exclusively. +function sessionsInUseLockName(email: string): string { + return `${sessionsLockName(email)}::in-use` +} + +interface InUseHold { + email: string + released: boolean + release?: () => void + done?: Promise +} + +let inUse: InUseHold | undefined + +// Resolves once the hold is granted, which waits for the sweep another tab is running. A +// request the browser refuses (a document that is not fully active) resolves it too, without +// a hold: the tab reads its sessions unguarded, as it does where Web Locks do not exist, and +// never sits waiting for a grant that is not coming. +async function holdSessionsInUse(email: string): Promise { + const locks = webLocks() + if (!locks || inUse?.email === email) return + await releaseSessionsInUse() + const hold: InUseHold = { email, released: false } + inUse = hold + await new Promise((granted) => { + hold.done = locks + .request(sessionsInUseLockName(email), { mode: 'shared' }, () => { + granted() + return hold.released ? undefined : new Promise((resolve) => (hold.release = resolve)) + }) + .catch((e) => { + console.error('Could not hold the AI sessions this tab is reading', e) + if (inUse === hold) inUse = undefined + granted() + }) + }) +} + +// Resolves once the hold is let go of, so an exclusive request made next can be granted. +async function releaseSessionsInUse(): Promise { + const hold = inUse + if (!hold) return + inUse = undefined + hold.released = true + hold.release?.() + await hold.done?.catch(() => {}) +} + +// Deletes one expired session: its record first, so nothing plans a push for it afterwards, +// then its pieces. The pending key, written before the record and removed once every piece +// is gone, is what a later sweep finishes a failed deletion from. +async function sweepSession( + db: IDBPDatabase, + id: string, + email: string +): Promise { + try { + localStorage.setItem(retentionPendingPrefix(email) + id, '1') + } catch { + return + } + await deleteSessionRow(db, id, false) + await sessionSwept(id, email) + if (await deleteSessionPieces(id, email)) forgetRetentionPending(email, id) +} + +// Chats with their images, artifacts and attached files. False when any of them could not +// be deleted. +async function deleteSessionPieces(id: string, email: string): Promise { + const chats = await deleteSessionChats(id, email) + const artifacts = await deleteArtifactsForSession(id, email) + const files = await deleteItemsForSession(id) + return chats && artifacts && files +} + +// The workspace a session's retention comes from: persisted unsent drafts count by the one +// they are waiting on. +function retentionWorkspaceOf(session: Session): string | undefined { + return session.workspace_id ?? session.pending_workspace_id +} + +// Deletes this browser's copies of the sessions past their workspace's retention, and the +// pieces of the ones an earlier sweep could not finish (docs/ai-session-backups.md). Deleting +// one record at a time, without re-reading it, is safe only under the in-use lock held +// exclusively, granted exactly when no tab has the sessions loaded — hence the call site. +async function sweepExpiredSessions(email: string): Promise { + const locks = webLocks() + if (!locks || inUse) return + try { + await locks.request(sessionsInUseLockName(email), { ifAvailable: true }, async (idle) => { + if (!idle) return + // The flush and the restore run under this one: neither must see a session half + // deleted, or plan a push from it. + await locks.request(sessionsLockName(email), { ifAvailable: true }, async (mirror) => { + if (!mirror) return + const db = await sessionsDb.whenReady() + if (!db || db.name !== scopedKeyFor(SESSIONS_DB, email)) return + for (const id of retentionPending(email)) { + // A restore brought the session back: its pieces are that copy's now. + const back = (await db.getKey('sessions', id)) !== undefined + if (back || (await deleteSessionPieces(id, email))) forgetRetentionPending(email, id) + } + const stored = await db.getAll('sessions') + const remembered = rememberedRetention(email) + const now = Date.now() + const workspaces = new Set() + let expired = false + for (const s of stored) { + const ws = retentionWorkspaceOf(s) + if (ws === undefined) continue + workspaces.add(ws) + expired ||= isSessionExpired(s, remembered?.days[ws], now) + } + // Nothing to sweep in, or nothing old enough by an answer recent enough to be + // believed about that: this load costs no request. + const fresh = remembered !== undefined && now - remembered.at < RETENTION_STALE_MS + if (workspaces.size === 0 || (!expired && fresh)) return + const retention = await askRetention([...workspaces]) + // Asked and not told: the sessions wait for the next load rather than go on an + // answer this browser does not have. + if (!retention) return + rememberRetention(email, retention) + for (const s of stored) { + const ws = retentionWorkspaceOf(s) + if (ws === undefined || !isSessionExpired(s, retention[ws], Date.now())) continue + await sweepSession(db, s.id, email) + } + }) + }) + } catch (e) { + console.error('Failed to sweep the sessions past their retention', e) + } +} + // The single seam for "a workspace just changed — bring sessions back in sync." // Refresh the workspace list FIRST — both reconcile and the putSession guard // read it, so it must reflect the change before reconcile runs — then reconcile. @@ -715,6 +969,15 @@ export async function deleteSessionsForWorkspace(workspaceId: string): Promise { if (!BROWSER) return + // The retention sweep runs here and nowhere else: this tab holds none of the new user's + // sessions yet, and letting go of the hold it had leaves it holding none of anyone's. The + // new hold is taken before the sessions are read, so another tab's sweep never deletes + // what this tab is about to load, and one already running is waited for. + await releaseSessionsInUse() + if (email) { + await sweepExpiredSessions(email) + await holdSessionsInUse(email) + } await hydrateSessions({ dropTransients: prevEmail !== email }) // onUserChange also fires at registration time, before the email resolves — // that hydration is an empty no-op and must not clear the loading state. @@ -1197,9 +1460,10 @@ export async function importSessions(records: Session[], email: string): Promise const tx = db.transaction('sessions', 'readwrite') const existing = new Set((await tx.store.getAllKeys()).map(String)) let next = nextSessionNumber([...(await tx.store.getAll()), ...sessionState.sessions]) + const restoredAt = Date.now() for (const r of records) { if (existing.has(r.id) || deletedSessionIds.has(r.id)) continue - const record: Session = { ...r, name: `session-${next++}` } + const record: Session = { ...r, name: `session-${next++}`, restoredAt } delete record.transient delete record.workspace_root_id ensureSessionRootId(record) diff --git a/frontend/src/lib/components/sessions/sessionStateIndexedDb.test.ts b/frontend/src/lib/components/sessions/sessionStateIndexedDb.test.ts index b9882d045e..dddfe795f7 100644 --- a/frontend/src/lib/components/sessions/sessionStateIndexedDb.test.ts +++ b/frontend/src/lib/components/sessions/sessionStateIndexedDb.test.ts @@ -9,20 +9,30 @@ vi.mock('esm-env', async (importOriginal) => ({ })) // Spy on the attached-file GC so we can assert lifecycle deletes clean it up. -const { deleteItemsForSessionMock } = vi.hoisted(() => ({ deleteItemsForSessionMock: vi.fn() })) +const { deleteItemsForSessionMock } = vi.hoisted(() => ({ + deleteItemsForSessionMock: vi.fn().mockResolvedValue(true) +})) vi.mock('../copilot/chat/files/attachedFilesDB', async (orig) => ({ ...(await orig()), deleteItemsForSession: deleteItemsForSessionMock })) const { deleteArtifactsForSessionMock } = vi.hoisted(() => ({ - deleteArtifactsForSessionMock: vi.fn() + deleteArtifactsForSessionMock: vi.fn().mockResolvedValue(true) })) vi.mock('../copilot/chat/artifacts/artifactsDB', async (orig) => ({ ...(await orig()), deleteArtifactsForSession: deleteArtifactsForSessionMock })) +const { deleteSessionChatsMock } = vi.hoisted(() => ({ + deleteSessionChatsMock: vi.fn().mockResolvedValue(true) +})) +vi.mock('../copilot/chat/HistoryManager.svelte', async (orig) => ({ + ...(await orig()), + deleteSessionChats: deleteSessionChatsMock +})) + // sessionState imports WorkspaceService; these tests don't touch the network. vi.mock('$lib/gen', async (orig) => { const actual = await orig() @@ -31,7 +41,8 @@ vi.mock('$lib/gen', async (orig) => { WorkspaceService: { ...actual.WorkspaceService, listUserWorkspaces: vi.fn().mockResolvedValue([]), - getSessionWorkspaceStatus: vi.fn().mockResolvedValue({}) + getSessionWorkspaceStatus: vi.fn().mockResolvedValue({}), + getSessionWorkspaceRetention: vi.fn().mockResolvedValue({}) } } }) @@ -75,6 +86,32 @@ function freshUser() { return asUser(`u${n++}@x.com`) } +// The Web Locks API, which the node test environment lacks: `holders` counts the shared holds +// on each name across tabs, against which an exclusive request made if available is not granted. +function installLocks(holders: Map): void { + if (typeof navigator === 'undefined') { + Object.defineProperty(globalThis, 'navigator', { value: {}, configurable: true }) + } + Object.defineProperty(navigator, 'locks', { + value: { + request: async (name: string, ...rest: unknown[]) => { + const run = rest[rest.length - 1] as (lock: unknown) => Promise + const options = (rest.length > 1 ? rest[0] : {}) as LockOptions + if (options.mode === 'shared') { + holders.set(name, (holders.get(name) ?? 0) + 1) + try { + return await run({}) + } finally { + holders.set(name, (holders.get(name) ?? 1) - 1) + } + } + return run(options.ifAvailable && (holders.get(name) ?? 0) > 0 ? null : {}) + } + }, + configurable: true + }) +} + // Hydration is fire-and-forget off the user store, so it can land after the test body // has populated sessionState.sessions and overwrite it with what the DB held at read // time; `hydrated` flips once the read has been applied. The logout is load-bearing: @@ -679,6 +716,93 @@ describe('sessionState IndexedDB persistence', () => { deleteSession('draftRec') }) + it('sweeps sessions past their workspace retention when a tab loads alone', async () => { + const user = freshUser() + usersWorkspaceStore.set({ + email: user.email, + workspaces: [ + { id: 'kept-ws', name: 'kept', disabled: false }, + { id: 'other-ws', name: 'other', disabled: false } + ] as never + }) + // The sweep runs only where Web Locks exist, and only as a tab loads: `login` is one. + const holders = new Map() + installLocks(holders) + const inUse = `wm-ai-sessions-mirror::${user.email}::in-use` + const otherTab = (n: number) => holders.set(inUse, (holders.get(inUse) ?? 0) + n) + await login(user) + const day = 24 * 60 * 60 * 1000 + const old = Date.now() - 31 * day + const stale = (id: string, over: Partial = {}) => + session({ id, createdAt: old, lastActivityAt: old, workspace_id: 'kept-ws', ...over }) + // Archived or not, a session is judged by its own last activity; one read a day ago + // stays, as do one restored here a day ago whatever the backup's time and one in a + // workspace without retention. + await putSession(stale('stale')) + await putSession(stale('stale-archived', { archived: true })) + await putSession(stale('read-lately', { lastActivityAt: Date.now() - day })) + await putSession(stale('restored-lately', { restoredAt: Date.now() - day })) + await putSession(stale('elsewhere', { workspace_id: 'other-ws' })) + + const retentionMock = vi.mocked(WorkspaceService.getSessionWorkspaceRetention) + let told: Record = { 'kept-ws': 30 } + retentionMock.mockImplementation(async () => told as never) + // The sweep believes a remembered answer for a day, so ageing it is how a later load + // is made to ask again. + const forgetWhenAsked = () => { + const key = `windmill_sessions_retention_days::${user.email}` + const remembered = JSON.parse(localStorage.getItem(key) ?? '{}') + localStorage.setItem(key, JSON.stringify({ ...remembered, at: Date.now() - 2 * day })) + } + const stored = async () => { + const db = await openDB(`windmill-sessions::${user.email}`, 1) + const ids = ((await db.getAll('sessions' as never)) as Session[]).map((s) => s.id) + db.close() + return ids.sort() + } + const chatDeletions = (id: string) => + deleteSessionChatsMock.mock.calls.filter(([sid, email]) => sid === id && email === user.email) + + // While another tab has the sessions loaded, nothing is swept. + otherTab(1) + await rehydrate(user) + expect(await stored()).toContain('stale') + expect(chatDeletions('stale')).toHaveLength(0) + otherTab(-1) + + // The retention is cleared when the sweep asks: what the server says then is what + // deletes, and a browser that remembered one deletes nothing on it. + told = {} + await rehydrate(user) + expect(await stored()).toContain('stale') + expect(chatDeletions('stale')).toHaveLength(0) + told = { 'kept-ws': 30 } + forgetWhenAsked() + + // The chats of the first expired session the sweep reaches, `stale` by key order, + // cannot be deleted this time. + deleteSessionChatsMock.mockResolvedValueOnce(false) + await rehydrate(user) + expect(await stored()).toEqual(['elsewhere', 'read-lately', 'restored-lately']) + const pending = (id: string) => + localStorage.getItem(`windmill_sessions_retention_pending::${user.email}::${id}`) + expect(chatDeletions('stale-archived')).toHaveLength(1) + expect(pending('stale')).toBe('1') + expect(pending('stale-archived')).toBeNull() + + // The next load finishes what that deletion left, with nothing else to sweep. + await rehydrate(user) + expect(pending('stale')).toBeNull() + expect(chatDeletions('stale')).toHaveLength(2) + + // A swept session is not tombstoned: the backup another device pushed to brings it back. + await importSessions([stale('stale')], user.email) + expect(await stored()).toContain('stale') + // Both are shared with the tests that follow, which expect neither. + retentionMock.mockResolvedValue({} as never) + Object.defineProperty(navigator, 'locks', { value: undefined, configurable: true }) + }) + it('clears the in-memory list on logout', async () => { const user = freshUser() await login(user) diff --git a/frontend/src/lib/components/workspaceSettings/AISettings.svelte b/frontend/src/lib/components/workspaceSettings/AISettings.svelte index a1dcc76758..14d27cab18 100644 --- a/frontend/src/lib/components/workspaceSettings/AISettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/AISettings.svelte @@ -693,7 +693,7 @@
    From e8078f2a963166b09849650424583f5dcfd28a84 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 16 Sep 2026 09:58:29 +0200 Subject: [PATCH 139/144] fix: dispatch workflow-as-code tasks from a deployed flow's inline step (#11146) * fix: dispatch workflow-as-code tasks from a deployed flow's inline step * fix: give a workflow-as-code task its own result-cache key * fix: key a cached workflow-as-code task on its name and arguments * fix: hash a cached workflow-as-code task's arguments like any job's * chore: regenerate system prompts for the task cache_ttl docs * fix: key a cached workflow-as-code task on its step key, not its name * fix: key a cached workflow-as-code task on a fingerprint of its code * fix: keep the task() doc attached to task() * fix: key a cached inline task on its step key and the workflow input * docs: cache_ttl has no effect on a taskFlow target --- backend/tests/bun_jobs.rs | 94 +++++++++++++++++++ backend/tests/fixtures/wac_flow_script.sql | 53 +++++++++++ backend/windmill-worker/src/bun_executor.rs | 64 +++++++++---- backend/windmill-worker/src/common.rs | 30 +++++- backend/windmill-worker/src/worker.rs | 2 +- backend/windmill-worker/src/worker_flow.rs | 3 +- cli/src/guidance/skills.gen.ts | 23 +++++ python-client/wmill/wmill/client.py | 8 ++ system_prompts/auto-generated/prompts.ts | 23 +++++ system_prompts/auto-generated/script.md | 8 ++ system_prompts/auto-generated/sdks/python.md | 8 ++ .../auto-generated/sdks/wac-python.md | 8 ++ .../auto-generated/sdks/wac-typescript.md | 7 ++ .../skills/write-script-python3/SKILL.md | 8 ++ .../skills/write-workflow-as-code/SKILL.md | 15 +++ typescript-client/client.ts | 7 ++ 16 files changed, 341 insertions(+), 20 deletions(-) create mode 100644 backend/tests/fixtures/wac_flow_script.sql diff --git a/backend/tests/bun_jobs.rs b/backend/tests/bun_jobs.rs index 590963b291..63d5341ba3 100644 --- a/backend/tests/bun_jobs.rs +++ b/backend/tests/bun_jobs.rs @@ -1209,6 +1209,100 @@ export function main() { Ok(()) } +/// A deployed flow runs an inline step as the `flow_node` its deploy rewrote it into, +/// a `FlowScript` job rather than the preview job the editor runs. A workflow-as-code +/// step's `task()` children must dispatch from that kind too, as re-runs of the same +/// node, or the step passes its editor test and fails once deployed. +/// +/// The step is cached: a child that shared the parent's result-cache key would hand +/// its own result (`10`) back to the parent on resume, in place of the workflow's. +#[sqlx::test(fixtures("base", "wac_flow_script"))] +async fn test_bun_wac_task_dispatch_from_flow_script(db: Pool) -> anyhow::Result<()> { + use windmill_common::flows::FlowNodeId; + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let node = FlowNodeId(3000000000000011); + let job = RunJob::from(JobPayload::FlowScript { + id: node, + path: "f/system/wac_flow_script/a".to_string(), + language: ScriptLang::Bun, + cache_ttl: Some(60), + cache_ignore_s3_path: None, + dedicated_worker: None, + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default(), + }) + .arg("n", serde_json::json!(5)) + .run_until_complete(&db, false, port) + .await; + + assert_eq!( + job.json_result().unwrap(), + serde_json::json!({"doubled": 10}) + ); + + let children: Vec<(String, Option, Option)> = sqlx::query_as( + "SELECT kind::text, runnable_id, cache_ttl FROM v2_job WHERE parent_job = $1", + ) + .bind(job.id) + .fetch_all(&db) + .await?; + assert_eq!( + children, + vec![("flowscript".to_string(), Some(node.0), None)], + "the task child re-runs the parent's flow node, outside the result cache" + ); + Ok(()) +} + +/// `task(fn, { cache_ttl })` on an inline task of a deployed flow's step: the child runs +/// the parent's code with the parent's arguments, so its result-cache key carries its +/// step key, or the parent and every sibling would read its result back as their own. +#[sqlx::test(fixtures("base", "wac_flow_script"))] +async fn test_bun_wac_inline_task_cache_is_per_task(db: Pool) -> anyhow::Result<()> { + use windmill_common::flows::FlowNodeId; + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let payload = || JobPayload::FlowScript { + id: FlowNodeId(3000000000000012), + path: "f/system/wac_flow_script/a".to_string(), + language: ScriptLang::Bun, + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default(), + }; + + let mut children_from_cache = Vec::new(); + for _ in 0..2 { + let job = RunJob::from(payload()) + .arg("n", serde_json::json!(5)) + .run_until_complete(&db, false, port) + .await; + assert_eq!( + job.json_result().unwrap(), + serde_json::json!({"doubled": 10, "tripled": 15}) + ); + let from_cache: i64 = sqlx::query_scalar( + "SELECT count(*) FROM job_logs l JOIN v2_job j ON j.id = l.job_id \ + WHERE j.parent_job = $1 AND l.logs LIKE '%found in cache%'", + ) + .bind(job.id) + .fetch_one(&db) + .await?; + children_from_cache.push(from_cache); + } + assert_eq!( + children_from_cache, + vec![0, 2], + "the second run serves each task from its own cache entry" + ); + Ok(()) +} + // ============================================================================ // Environment Variable Tests // ============================================================================ diff --git a/backend/tests/fixtures/wac_flow_script.sql b/backend/tests/fixtures/wac_flow_script.sql new file mode 100644 index 0000000000..1b780c6cf4 --- /dev/null +++ b/backend/tests/fixtures/wac_flow_script.sql @@ -0,0 +1,53 @@ +-- A deployed flow whose inline bun step is workflow-as-code calling task(), in the +-- shape the deploy leaves behind: the RawScript module rewritten into a flow_node that +-- the step then runs as a FlowScript job. No lock, so the worker resolves +-- windmill-client at run time like the other bun fixtures. +INSERT INTO public.flow(workspace_id, summary, description, path, versions, schema, value, edited_by) VALUES ( +'test-workspace', '', '', +'f/system/wac_flow_script', +'{}', +'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"n":{"type":"integer","description":""}},"required":[],"type":"object"}', +'{"modules":[{"id":"a","value":{"type":"flowscript","id":3000000000000011,"language":"bun","input_transforms":{"n":{"expr":"flow_input.n","type":"javascript"}}}}]}', +'system' +); + +INSERT INTO public.flow_node(id, workspace_id, path, hash_v2, lock, code) VALUES ( +3000000000000011, +'test-workspace', +'f/system/wac_flow_script', +'0000000000000000000000000000000000000000000000000000000000000011', +NULL, +E'import { workflow, task } from "windmill-client"; + +const double = task(async (n: number) => { + return n * 2; +}); + +export const main = workflow(async (n: number) => { + const d = await double(n); + return { doubled: d }; +});' +); + +-- The same flow's step with two tasks that cache their own result. +INSERT INTO public.flow_node(id, workspace_id, path, hash_v2, lock, code) VALUES ( +3000000000000012, +'test-workspace', +'f/system/wac_flow_script', +'0000000000000000000000000000000000000000000000000000000000000012', +NULL, +E'import { workflow, task } from "windmill-client"; + +const double = task(async (n: number) => { + return n * 2; +}, { cache_ttl: 60 }); +const triple = task(async (n: number) => { + return n * 3; +}, { cache_ttl: 60 }); + +export const main = workflow(async (n: number) => { + const d = await double(n); + const t = await triple(n); + return { doubled: d, tripled: t }; +});' +); diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index 7d45cab178..117fbda086 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -2874,15 +2874,17 @@ pub async fn handle_wac_v2_output( .collect(); // Resolve job_payload once (same for all children since they re-run - // the parent script) + // the parent script). The step's cache setting is for the workflow's + // result; a task is cached only through its own `cache_ttl` option, + // under a key of its own (see `cached_result_path`). let job_payload_template = match job.kind { JobKind::Script => { if let Some(hash) = job.runnable_id { Ok(JobPayload::ScriptHash { hash, path: job.runnable_path.clone().unwrap_or_default(), - cache_ttl: job.cache_ttl, - cache_ignore_s3_path: job.cache_ignore_s3_path, + cache_ttl: None, + cache_ignore_s3_path: None, dedicated_worker: None, language: job.script_lang.unwrap_or(ScriptLang::Bun), priority: job.priority, @@ -2897,6 +2899,27 @@ pub async fn handle_wac_v2_output( )) } } + // A deployed flow runs an inline step as the `flow_node` its deploy + // rewrote it into; the child re-runs that node the way a `Script` + // child re-runs its hash, so `runnable_id` (the checkpoint's source + // hash) stays the same across parent and children. + JobKind::FlowScript => { + if let Some(id) = job.runnable_id { + Ok(JobPayload::FlowScript { + id: windmill_common::flows::FlowNodeId(id.0), + path: job.runnable_path.clone().unwrap_or_default(), + language: job.script_lang.unwrap_or(ScriptLang::Bun), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + concurrency_settings: ConcurrencySettings::default(), + }) + } else { + Err(error::Error::internal_err( + "WAC v2 FlowScript job missing runnable_id".to_string(), + )) + } + } JobKind::Preview => { let row: Option<(Option, Option)> = sqlx::query_as( "SELECT raw_code, raw_lock FROM v2_job WHERE id = $1 AND workspace_id = $2", @@ -2912,8 +2935,8 @@ pub async fn handle_wac_v2_output( hash: None, language: job.script_lang.unwrap_or(ScriptLang::Bun), lock: lock, - cache_ttl: job.cache_ttl, - cache_ignore_s3_path: job.cache_ignore_s3_path, + cache_ttl: None, + cache_ignore_s3_path: None, dedicated_worker: None, concurrency_settings: ConcurrencySettingsWithCustom::default(), debouncing_settings: DebouncingSettings::default(), @@ -3012,6 +3035,12 @@ pub async fn handle_wac_v2_output( let mut pushed_ids: Vec = Vec::with_capacity(num_steps); let push_result: error::Result<()> = async { for (step, (_, child_uuid)) in steps.iter().zip(job_ids.iter()) { + // A task with a runnable of its own (a deployed script or flow) queues + // at that runnable's priority; any other task is the parent's code and + // queues at the parent's. + let own_runnable = matches!(step.dispatch_type.as_str(), "script" | "flow") + && !step.script.starts_with("./"); + // Resolve job payload based on dispatch_type let (job_payload, child_args, is_external, on_behalf_of) = match step.dispatch_type.as_str() { @@ -3025,8 +3054,8 @@ pub async fn handle_wac_v2_output( hash: None, language: module.language, lock: module.lock, - cache_ttl: job.cache_ttl, - cache_ignore_s3_path: job.cache_ignore_s3_path, + cache_ttl: None, + cache_ignore_s3_path: None, dedicated_worker: None, concurrency_settings: ConcurrencySettingsWithCustom::default(), debouncing_settings: DebouncingSettings::default(), @@ -3110,7 +3139,8 @@ pub async fn handle_wac_v2_output( let mut job_payload = job_payload; if let Some(cache_ttl) = step.cache_ttl { match &mut job_payload { - JobPayload::ScriptHash { cache_ttl: ref mut ct, .. } => { + JobPayload::ScriptHash { cache_ttl: ref mut ct, .. } + | JobPayload::FlowScript { cache_ttl: ref mut ct, .. } => { *ct = Some(cache_ttl) } JobPayload::Code(ref mut code) => code.cache_ttl = Some(cache_ttl), @@ -3122,7 +3152,8 @@ pub async fn handle_wac_v2_output( || step.concurrency_time_window_s.is_some() { match &mut job_payload { - JobPayload::ScriptHash { concurrency_settings: ref mut cs, .. } => { + JobPayload::ScriptHash { concurrency_settings: ref mut cs, .. } + | JobPayload::FlowScript { concurrency_settings: ref mut cs, .. } => { if let Some(limit) = step.concurrent_limit { cs.concurrent_limit = Some(limit); } @@ -3188,13 +3219,14 @@ pub async fn handle_wac_v2_output( job.visible_to_owner, step.tag.clone().or_else(|| Some(job.tag.clone())), step.timeout.or(job.timeout), - None, // flow_step_id - step.priority, // priority_override - None, // authed - false, // running - None, // end_user_email - None, // trigger - None, // suspended_mode + None, // flow_step_id + step.priority + .or(if own_runnable { None } else { job.priority }), + None, // authed + false, // running + None, // end_user_email + None, // trigger + None, // suspended_mode ) .await?; diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index 0a5deb52d2..9c012fd117 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -1559,7 +1559,7 @@ pub async fn cached_result_path( client: &AuthedClient, job: &MiniPulledJob, raw_data: Option<&RawData>, -) -> String { +) -> windmill_common::error::Result { let mut hasher = sha2::Sha256::new(); hasher.update(&[job.kind as u8]); if let Some(ScriptHash(hash)) = job.runnable_id { @@ -1574,6 +1574,13 @@ pub async fn cached_result_path( _ => {} } } + // A workflow-as-code task child runs its parent's code with the parent's + // arguments; the step it executes is what tells its result from the parent's + // and from its siblings'. + if let Some(step_key) = wac_executing_key(db, job).await? { + hasher.update(b"wac_step:"); + hasher.update(step_key.as_bytes()); + } hash_args( db, client, @@ -1584,7 +1591,26 @@ pub async fn cached_result_path( job.cache_ignore_s3_path.unwrap_or(false), ) .await; - format!("g/results/{:064x}", hasher.finalize()) + Ok(format!("g/results/{:064x}", hasher.finalize())) +} + +/// The checkpoint step key a workflow-as-code parent seeded for this child at push +/// time; `None` for any job that is not such a child. +async fn wac_executing_key( + db: &DB, + job: &MiniPulledJob, +) -> windmill_common::error::Result> { + if job.parent_job.is_none() || job.flow_step_id.is_some() { + return Ok(None); + } + let key: Option> = sqlx::query_scalar( + "SELECT workflow_as_code_status->'_checkpoint'->>'_executing_key' \ + FROM v2_job_status WHERE id = $1", + ) + .bind(job.id) + .fetch_optional(db) + .await?; + Ok(key.flatten()) } #[cfg(feature = "parquet")] diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 467b8dbf19..56b4b26fb0 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -4645,7 +4645,7 @@ pub async fn handle_queued_job( let cached_res_path = if job.cache_ttl.is_some() { match conn { Connection::Sql(db) => { - Some(cached_result_path(db, &client, &job, preview_data.as_ref()).await) + Some(cached_result_path(db, &client, &job, preview_data.as_ref()).await?) } Connection::Http(_) => None, } diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index cbda50c7bc..ed543bd517 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -2005,7 +2005,8 @@ pub async fn update_flow_status_after_job_completion_internal( if flow_job.cache_ttl.is_some() && success { let flow = RawData::Flow(flow_data.clone()); - let cached_res_path = cached_result_path(db, client, &flow_job, Some(&flow)).await; + let cached_res_path = + cached_result_path(db, client, &flow_job, Some(&flow)).await?; save_in_cache( db, diff --git a/cli/src/guidance/skills.gen.ts b/cli/src/guidance/skills.gen.ts index 60fdd07124..f58092fdac 100644 --- a/cli/src/guidance/skills.gen.ts +++ b/cli/src/guidance/skills.gen.ts @@ -4608,6 +4608,14 @@ def parse_sql_client_name(name: str) -> tuple[str, Optional[str]] # it grows with both the width of the fan-out and \`\`attempts\`\`. Retries with # no \`\`delay\`\` all go out in a single round. # +# \`\`cache_ttl\`\` serves a previous result of the task for that many seconds +# instead of running it again. A task is keyed on its step key (its name and +# call order) and the workflow's input, not on the arguments it is called +# with, so cache one only when whether it runs, and what it receives, follow +# from the workflow's input alone. A \`\`task_script\`\` target is keyed on the +# arguments it is called with. It has no effect on a \`\`task_flow\`\` target, +# which keeps its flow's own cache policy. +# # Usage:: # # @task @@ -6742,6 +6750,13 @@ export interface TaskRetry { export interface TaskOptions { timeout?: number; tag?: string; + /** Seconds during which a previous result of this task is served instead of + * running it again. A task written inline in the workflow is keyed on its + * step key (its name and call order) and the workflow's input, not on the + * arguments it is called with, so cache one only when whether it runs, and + * what it receives, follow from the workflow's input alone. A \`taskScript\` + * target is keyed on the arguments it is called with. It has no effect on a + * \`taskFlow\` target, which keeps its flow's own cache policy. */ cache_ttl?: number; priority?: number; concurrency_limit?: number; @@ -6933,6 +6948,14 @@ def get_resume_urls(approver: str = None, flow_level: bool = None) -> dict # it grows with both the width of the fan-out and \`\`attempts\`\`. Retries with # no \`\`delay\`\` all go out in a single round. # +# \`\`cache_ttl\`\` serves a previous result of the task for that many seconds +# instead of running it again. A task is keyed on its step key (its name and +# call order) and the workflow's input, not on the arguments it is called +# with, so cache one only when whether it runs, and what it receives, follow +# from the workflow's input alone. A \`\`task_script\`\` target is keyed on the +# arguments it is called with. It has no effect on a \`\`task_flow\`\` target, +# which keeps its flow's own cache policy. +# # Usage:: # # @task diff --git a/python-client/wmill/wmill/client.py b/python-client/wmill/wmill/client.py index 918a16f2b0..dbc9d9b193 100644 --- a/python-client/wmill/wmill/client.py +++ b/python-client/wmill/wmill/client.py @@ -3327,6 +3327,14 @@ def task( it grows with both the width of the fan-out and ``attempts``. Retries with no ``delay`` all go out in a single round. + ``cache_ttl`` serves a previous result of the task for that many seconds + instead of running it again. A task is keyed on its step key (its name and + call order) and the workflow's input, not on the arguments it is called + with, so cache one only when whether it runs, and what it receives, follow + from the workflow's input alone. A ``task_script`` target is keyed on the + arguments it is called with. It has no effect on a ``task_flow`` target, + which keeps its flow's own cache policy. + Usage:: @task diff --git a/system_prompts/auto-generated/prompts.ts b/system_prompts/auto-generated/prompts.ts index 2a20e8f779..1e92c8e69b 100644 --- a/system_prompts/auto-generated/prompts.ts +++ b/system_prompts/auto-generated/prompts.ts @@ -2588,6 +2588,14 @@ def parse_sql_client_name(name: str) -> tuple[str, Optional[str]] # it grows with both the width of the fan-out and \`\`attempts\`\`. Retries with # no \`\`delay\`\` all go out in a single round. # +# \`\`cache_ttl\`\` serves a previous result of the task for that many seconds +# instead of running it again. A task is keyed on its step key (its name and +# call order) and the workflow's input, not on the arguments it is called +# with, so cache one only when whether it runs, and what it receives, follow +# from the workflow's input alone. A \`\`task_script\`\` target is keyed on the +# arguments it is called with. It has no effect on a \`\`task_flow\`\` target, +# which keeps its flow's own cache policy. +# # Usage:: # # @task @@ -2738,6 +2746,13 @@ export interface TaskRetry { export interface TaskOptions { timeout?: number; tag?: string; + /** Seconds during which a previous result of this task is served instead of + * running it again. A task written inline in the workflow is keyed on its + * step key (its name and call order) and the workflow's input, not on the + * arguments it is called with, so cache one only when whether it runs, and + * what it receives, follow from the workflow's input alone. A \`taskScript\` + * target is keyed on the arguments it is called with. It has no effect on a + * \`taskFlow\` target, which keeps its flow's own cache policy. */ cache_ttl?: number; priority?: number; concurrency_limit?: number; @@ -2929,6 +2944,14 @@ def get_resume_urls(approver: str = None, flow_level: bool = None) -> dict # it grows with both the width of the fan-out and \`\`attempts\`\`. Retries with # no \`\`delay\`\` all go out in a single round. # +# \`\`cache_ttl\`\` serves a previous result of the task for that many seconds +# instead of running it again. A task is keyed on its step key (its name and +# call order) and the workflow's input, not on the arguments it is called +# with, so cache one only when whether it runs, and what it receives, follow +# from the workflow's input alone. A \`\`task_script\`\` target is keyed on the +# arguments it is called with. It has no effect on a \`\`task_flow\`\` target, +# which keeps its flow's own cache policy. +# # Usage:: # # @task diff --git a/system_prompts/auto-generated/script.md b/system_prompts/auto-generated/script.md index 7883a8ba4b..f278a67fba 100644 --- a/system_prompts/auto-generated/script.md +++ b/system_prompts/auto-generated/script.md @@ -2728,6 +2728,14 @@ def parse_sql_client_name(name: str) -> tuple[str, Optional[str]] # it grows with both the width of the fan-out and ``attempts``. Retries with # no ``delay`` all go out in a single round. # +# ``cache_ttl`` serves a previous result of the task for that many seconds +# instead of running it again. A task is keyed on its step key (its name and +# call order) and the workflow's input, not on the arguments it is called +# with, so cache one only when whether it runs, and what it receives, follow +# from the workflow's input alone. A ``task_script`` target is keyed on the +# arguments it is called with. It has no effect on a ``task_flow`` target, +# which keeps its flow's own cache policy. +# # Usage:: # # @task diff --git a/system_prompts/auto-generated/sdks/python.md b/system_prompts/auto-generated/sdks/python.md index 4aacf4d1b4..4f727c7d45 100644 --- a/system_prompts/auto-generated/sdks/python.md +++ b/system_prompts/auto-generated/sdks/python.md @@ -672,6 +672,14 @@ def parse_sql_client_name(name: str) -> tuple[str, Optional[str]] # it grows with both the width of the fan-out and ``attempts``. Retries with # no ``delay`` all go out in a single round. # +# ``cache_ttl`` serves a previous result of the task for that many seconds +# instead of running it again. A task is keyed on its step key (its name and +# call order) and the workflow's input, not on the arguments it is called +# with, so cache one only when whether it runs, and what it receives, follow +# from the workflow's input alone. A ``task_script`` target is keyed on the +# arguments it is called with. It has no effect on a ``task_flow`` target, +# which keeps its flow's own cache policy. +# # Usage:: # # @task diff --git a/system_prompts/auto-generated/sdks/wac-python.md b/system_prompts/auto-generated/sdks/wac-python.md index 816ea4959b..a8e98a9d4f 100644 --- a/system_prompts/auto-generated/sdks/wac-python.md +++ b/system_prompts/auto-generated/sdks/wac-python.md @@ -58,6 +58,14 @@ def get_resume_urls(approver: str = None, flow_level: bool = None) -> dict # it grows with both the width of the fan-out and ``attempts``. Retries with # no ``delay`` all go out in a single round. # +# ``cache_ttl`` serves a previous result of the task for that many seconds +# instead of running it again. A task is keyed on its step key (its name and +# call order) and the workflow's input, not on the arguments it is called +# with, so cache one only when whether it runs, and what it receives, follow +# from the workflow's input alone. A ``task_script`` target is keyed on the +# arguments it is called with. It has no effect on a ``task_flow`` target, +# which keeps its flow's own cache policy. +# # Usage:: # # @task diff --git a/system_prompts/auto-generated/sdks/wac-typescript.md b/system_prompts/auto-generated/sdks/wac-typescript.md index 66eecfe761..608d75d18c 100644 --- a/system_prompts/auto-generated/sdks/wac-typescript.md +++ b/system_prompts/auto-generated/sdks/wac-typescript.md @@ -34,6 +34,13 @@ export interface TaskRetry { export interface TaskOptions { timeout?: number; tag?: string; + /** Seconds during which a previous result of this task is served instead of + * running it again. A task written inline in the workflow is keyed on its + * step key (its name and call order) and the workflow's input, not on the + * arguments it is called with, so cache one only when whether it runs, and + * what it receives, follow from the workflow's input alone. A `taskScript` + * target is keyed on the arguments it is called with. It has no effect on a + * `taskFlow` target, which keeps its flow's own cache policy. */ cache_ttl?: number; priority?: number; concurrency_limit?: number; diff --git a/system_prompts/auto-generated/skills/write-script-python3/SKILL.md b/system_prompts/auto-generated/skills/write-script-python3/SKILL.md index 8100eebc25..1ccb376a9f 100644 --- a/system_prompts/auto-generated/skills/write-script-python3/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-python3/SKILL.md @@ -857,6 +857,14 @@ def parse_sql_client_name(name: str) -> tuple[str, Optional[str]] # it grows with both the width of the fan-out and ``attempts``. Retries with # no ``delay`` all go out in a single round. # +# ``cache_ttl`` serves a previous result of the task for that many seconds +# instead of running it again. A task is keyed on its step key (its name and +# call order) and the workflow's input, not on the arguments it is called +# with, so cache one only when whether it runs, and what it receives, follow +# from the workflow's input alone. A ``task_script`` target is keyed on the +# arguments it is called with. It has no effect on a ``task_flow`` target, +# which keeps its flow's own cache policy. +# # Usage:: # # @task diff --git a/system_prompts/auto-generated/skills/write-workflow-as-code/SKILL.md b/system_prompts/auto-generated/skills/write-workflow-as-code/SKILL.md index c41bd54f04..8e5ab1f06c 100644 --- a/system_prompts/auto-generated/skills/write-workflow-as-code/SKILL.md +++ b/system_prompts/auto-generated/skills/write-workflow-as-code/SKILL.md @@ -277,6 +277,13 @@ export interface TaskRetry { export interface TaskOptions { timeout?: number; tag?: string; + /** Seconds during which a previous result of this task is served instead of + * running it again. A task written inline in the workflow is keyed on its + * step key (its name and call order) and the workflow's input, not on the + * arguments it is called with, so cache one only when whether it runs, and + * what it receives, follow from the workflow's input alone. A `taskScript` + * target is keyed on the arguments it is called with. It has no effect on a + * `taskFlow` target, which keeps its flow's own cache policy. */ cache_ttl?: number; priority?: number; concurrency_limit?: number; @@ -468,6 +475,14 @@ def get_resume_urls(approver: str = None, flow_level: bool = None) -> dict # it grows with both the width of the fan-out and ``attempts``. Retries with # no ``delay`` all go out in a single round. # +# ``cache_ttl`` serves a previous result of the task for that many seconds +# instead of running it again. A task is keyed on its step key (its name and +# call order) and the workflow's input, not on the arguments it is called +# with, so cache one only when whether it runs, and what it receives, follow +# from the workflow's input alone. A ``task_script`` target is keyed on the +# arguments it is called with. It has no effect on a ``task_flow`` target, +# which keeps its flow's own cache policy. +# # Usage:: # # @task diff --git a/typescript-client/client.ts b/typescript-client/client.ts index 7b10c732be..dd2123e746 100644 --- a/typescript-client/client.ts +++ b/typescript-client/client.ts @@ -1712,6 +1712,13 @@ export interface TaskRetry { export interface TaskOptions { timeout?: number; tag?: string; + /** Seconds during which a previous result of this task is served instead of + * running it again. A task written inline in the workflow is keyed on its + * step key (its name and call order) and the workflow's input, not on the + * arguments it is called with, so cache one only when whether it runs, and + * what it receives, follow from the workflow's input alone. A `taskScript` + * target is keyed on the arguments it is called with. It has no effect on a + * `taskFlow` target, which keeps its flow's own cache policy. */ cache_ttl?: number; priority?: number; concurrency_limit?: number; From b9b5988ebdf3edd75add445282977c516ef5ed51 Mon Sep 17 00:00:00 2001 From: Guilhem Date: Wed, 16 Sep 2026 10:42:47 +0200 Subject: [PATCH 140/144] fix: keep sidebar confirmation dialogs from being confined to the rail (#11158) Co-authored-by: Claude Fable 5.1 --- frontend/src/lib/components/home/ItemsList.svelte | 5 ++++- .../components/sidebar/DeleteForkedWorkspaceModal.svelte | 3 +++ frontend/src/lib/components/sidebar/SettingsMenu.svelte | 3 +++ frontend/src/lib/components/sidebar/SidebarContent.svelte | 3 +++ frontend/src/routes/(root)/(logged)/+layout.svelte | 6 +++++- 5 files changed, 18 insertions(+), 2 deletions(-) diff --git a/frontend/src/lib/components/home/ItemsList.svelte b/frontend/src/lib/components/home/ItemsList.svelte index 35b1c5a6f3..7038a316fa 100644 --- a/frontend/src/lib/components/home/ItemsList.svelte +++ b/frontend/src/lib/components/home/ItemsList.svelte @@ -2107,8 +2107,11 @@ } } + /* `backwards`, not `both`: the fill hides rows until their staggered start, but a forwards + fill would keep `transform` animated afterwards and confine the row's remove confirmation + (`position: fixed`) to the row. The `to` keyframe equals the row's resting style. */ .wm-imported > :global(*) { - animation: wm-row-in 260ms ease-out both; + animation: wm-row-in 260ms ease-out backwards; animation-delay: 320ms; } .wm-imported > :global(*:nth-child(1)) { diff --git a/frontend/src/lib/components/sidebar/DeleteForkedWorkspaceModal.svelte b/frontend/src/lib/components/sidebar/DeleteForkedWorkspaceModal.svelte index ff049ceddb..3a73758146 100644 --- a/frontend/src/lib/components/sidebar/DeleteForkedWorkspaceModal.svelte +++ b/frontend/src/lib/components/sidebar/DeleteForkedWorkspaceModal.svelte @@ -151,8 +151,11 @@ {#if currentWsIsFork} + { diff --git a/frontend/src/lib/components/sidebar/SettingsMenu.svelte b/frontend/src/lib/components/sidebar/SettingsMenu.svelte index 857e422bdd..e0951f37cd 100644 --- a/frontend/src/lib/components/sidebar/SettingsMenu.svelte +++ b/frontend/src/lib/components/sidebar/SettingsMenu.svelte @@ -396,8 +396,11 @@ {/snippet} + { diff --git a/frontend/src/lib/components/sidebar/SidebarContent.svelte b/frontend/src/lib/components/sidebar/SidebarContent.svelte index 7fa161dd2d..b43bc2988e 100644 --- a/frontend/src/lib/components/sidebar/SidebarContent.svelte +++ b/frontend/src/lib/components/sidebar/SidebarContent.svelte @@ -737,8 +737,11 @@
    + { diff --git a/frontend/src/routes/(root)/(logged)/+layout.svelte b/frontend/src/routes/(root)/(logged)/+layout.svelte index 225b9443d3..c82e8b9d54 100644 --- a/frontend/src/routes/(root)/(logged)/+layout.svelte +++ b/frontend/src/routes/(root)/(logged)/+layout.svelte @@ -1515,8 +1515,12 @@ } } + /* No forwards fill: a filled animation keeps `transform` animated after it ends, which makes + the rail the containing block for every `position: fixed` descendant — the confirmation + dialogs opened from the settings menu would be confined to the rail's column. The `to` + keyframe equals the rail's resting style, so nothing changes visually when the fill drops. */ :global(#sidebar.wm-sidebar-in) { - animation: wm-sidebar-in 500ms ease-out both; + animation: wm-sidebar-in 500ms ease-out; } @media (prefers-reduced-motion: reduce) { From 57a99f66a88f195cac8f583b59d69d627cc1ec1d Mon Sep 17 00:00:00 2001 From: hugocasa Date: Wed, 16 Sep 2026 10:45:01 +0200 Subject: [PATCH 141/144] feat: rename saved agents from the agent editor and flag broken links (#11147) * feat: list flows that link a saved agent and flag broken agent links * feat: rename saved agents from the agent editor and repoint the flow * fix: show an unreadable linked agent as not accessible, not missing * fix: address review nits on agent rename and missing-agent state * fix: open content search above modals and keep Escape for it * fix: register content search on the opener's overlay stack * docs: scope the global search z-index comment to the bases it clears * refactor: show linked agents' rename warning as for scripts and flows * fix: keep the failed-lookup rename warning to resources --- ...2e38967680a77316ee9a53e67d70d7df9f63.json} | 4 +- ...1892fa3899934e7e0e51ad23bfcd2d6d3d08.json} | 4 +- ...4250441863ab47c345d9685f53c7e224b4887.json | 15 +++ ...c6ac927bd0e6b0f383626e673225065ee90bf.json | 23 ++++ ...f3bee9b93ba0387f566ad7f502cbee818296e.json | 15 --- ...eea76ab0226a514d25712a733e01add508b71.json | 16 +++ ...d_agents_as_runnable_dependencies.down.sql | 9 ++ ...ked_agents_as_runnable_dependencies.up.sql | 21 +++ backend/summarized_schema.txt | 2 +- backend/windmill-api-flows/src/flows.rs | 32 ++++- .../windmill-api-workspaces/src/workspaces.rs | 4 +- backend/windmill-api/openapi.yaml | 19 +++ .../windmill-worker/src/worker_lockfiles.rs | 11 ++ .../src/lib/components/FlowBuilder.svelte | 4 +- frontend/src/lib/components/Path.svelte | 69 ++++++---- .../lib/components/flows/FlowEditor.svelte | 2 + .../lib/components/flows/agentDraft.svelte.ts | 52 +++++--- .../flows/content/AgentEditorHost.svelte | 40 +++++- .../flows/content/AgentEditorModal.svelte | 28 +++- .../flows/content/AgentResourceBar.svelte | 125 +++++++++++++----- .../flows/linkedAgentDrafts.test.ts | 33 +++++ .../lib/components/flows/linkedAgentDrafts.ts | 21 ++- .../search/GlobalSearchModal.svelte | 31 ++++- frontend/src/lib/zIndexes.ts | 4 + .../src/routes/(root)/(logged)/+layout.svelte | 7 +- .../(root)/(logged)/resources/+page.svelte | 14 +- 26 files changed, 482 insertions(+), 123 deletions(-) rename backend/.sqlx/{query-00c0ae12b19ba495f307f0ce6b4833947c5b3fe45826fc5468e326d171d95236.json => query-0c49b098051900b834cb791e37af2e38967680a77316ee9a53e67d70d7df9f63.json} (78%) rename backend/.sqlx/{query-dc5eeb7b7bf0b7217ef66eb950ab7e9cf578bba7bd1eec981526be4067bcb314.json => query-139e153d1ebe584e878d3b2569551892fa3899934e7e0e51ad23bfcd2d6d3d08.json} (76%) create mode 100644 backend/.sqlx/query-2aa87574b437f0e29991696564c4250441863ab47c345d9685f53c7e224b4887.json create mode 100644 backend/.sqlx/query-46e65196c2a4f07d171a22f1e45c6ac927bd0e6b0f383626e673225065ee90bf.json delete mode 100644 backend/.sqlx/query-a54e2334c365f90577f68ebefc8f3bee9b93ba0387f566ad7f502cbee818296e.json create mode 100644 backend/.sqlx/query-fa8c36eda6d4cb64b4ac5979cc4eea76ab0226a514d25712a733e01add508b71.json create mode 100644 backend/migrations/20260915111928_record_linked_agents_as_runnable_dependencies.down.sql create mode 100644 backend/migrations/20260915111928_record_linked_agents_as_runnable_dependencies.up.sql diff --git a/backend/.sqlx/query-00c0ae12b19ba495f307f0ce6b4833947c5b3fe45826fc5468e326d171d95236.json b/backend/.sqlx/query-0c49b098051900b834cb791e37af2e38967680a77316ee9a53e67d70d7df9f63.json similarity index 78% rename from backend/.sqlx/query-00c0ae12b19ba495f307f0ce6b4833947c5b3fe45826fc5468e326d171d95236.json rename to backend/.sqlx/query-0c49b098051900b834cb791e37af2e38967680a77316ee9a53e67d70d7df9f63.json index f950c5bf57..da25d2fcf8 100644 --- a/backend/.sqlx/query-00c0ae12b19ba495f307f0ce6b4833947c5b3fe45826fc5468e326d171d95236.json +++ b/backend/.sqlx/query-0c49b098051900b834cb791e37af2e38967680a77316ee9a53e67d70d7df9f63.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT f.path\n FROM workspace_runnable_dependencies wru \n JOIN flow f\n ON wru.flow_path = f.path AND wru.workspace_id = f.workspace_id\n WHERE wru.runnable_path = $1 AND wru.runnable_is_flow = $2 AND wru.workspace_id = $3", + "query": "SELECT f.path\n FROM workspace_runnable_dependencies wru \n JOIN flow f\n ON wru.flow_path = f.path AND wru.workspace_id = f.workspace_id\n WHERE wru.runnable_path = $1 AND wru.runnable_is_flow = $2 AND NOT wru.runnable_is_agent AND wru.workspace_id = $3", "describe": { "columns": [ { @@ -20,5 +20,5 @@ false ] }, - "hash": "00c0ae12b19ba495f307f0ce6b4833947c5b3fe45826fc5468e326d171d95236" + "hash": "0c49b098051900b834cb791e37af2e38967680a77316ee9a53e67d70d7df9f63" } diff --git a/backend/.sqlx/query-dc5eeb7b7bf0b7217ef66eb950ab7e9cf578bba7bd1eec981526be4067bcb314.json b/backend/.sqlx/query-139e153d1ebe584e878d3b2569551892fa3899934e7e0e51ad23bfcd2d6d3d08.json similarity index 76% rename from backend/.sqlx/query-dc5eeb7b7bf0b7217ef66eb950ab7e9cf578bba7bd1eec981526be4067bcb314.json rename to backend/.sqlx/query-139e153d1ebe584e878d3b2569551892fa3899934e7e0e51ad23bfcd2d6d3d08.json index 37115e4ab7..780fb75ea9 100644 --- a/backend/.sqlx/query-dc5eeb7b7bf0b7217ef66eb950ab7e9cf578bba7bd1eec981526be4067bcb314.json +++ b/backend/.sqlx/query-139e153d1ebe584e878d3b2569551892fa3899934e7e0e51ad23bfcd2d6d3d08.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT DISTINCT f.path\n FROM workspace_runnable_dependencies wru \n JOIN flow f\n ON wru.flow_path = f.path AND wru.workspace_id = f.workspace_id\n WHERE wru.runnable_path LIKE $1 || '%' AND wru.runnable_is_flow = $2 AND wru.workspace_id = $3", + "query": "SELECT DISTINCT f.path\n FROM workspace_runnable_dependencies wru \n JOIN flow f\n ON wru.flow_path = f.path AND wru.workspace_id = f.workspace_id\n WHERE wru.runnable_path LIKE $1 || '%' AND wru.runnable_is_flow = $2 AND NOT wru.runnable_is_agent AND wru.workspace_id = $3", "describe": { "columns": [ { @@ -20,5 +20,5 @@ false ] }, - "hash": "dc5eeb7b7bf0b7217ef66eb950ab7e9cf578bba7bd1eec981526be4067bcb314" + "hash": "139e153d1ebe584e878d3b2569551892fa3899934e7e0e51ad23bfcd2d6d3d08" } diff --git a/backend/.sqlx/query-2aa87574b437f0e29991696564c4250441863ab47c345d9685f53c7e224b4887.json b/backend/.sqlx/query-2aa87574b437f0e29991696564c4250441863ab47c345d9685f53c7e224b4887.json new file mode 100644 index 0000000000..919d930f70 --- /dev/null +++ b/backend/.sqlx/query-2aa87574b437f0e29991696564c4250441863ab47c345d9685f53c7e224b4887.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace_runnable_dependencies (flow_path, runnable_path, script_hash, runnable_is_flow, runnable_is_agent, workspace_id, app_path)\n SELECT flow_path, runnable_path, script_hash, runnable_is_flow, runnable_is_agent, $1, app_path\n FROM workspace_runnable_dependencies\n WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "2aa87574b437f0e29991696564c4250441863ab47c345d9685f53c7e224b4887" +} diff --git a/backend/.sqlx/query-46e65196c2a4f07d171a22f1e45c6ac927bd0e6b0f383626e673225065ee90bf.json b/backend/.sqlx/query-46e65196c2a4f07d171a22f1e45c6ac927bd0e6b0f383626e673225065ee90bf.json new file mode 100644 index 0000000000..9b361b8f53 --- /dev/null +++ b/backend/.sqlx/query-46e65196c2a4f07d171a22f1e45c6ac927bd0e6b0f383626e673225065ee90bf.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT DISTINCT f.path\n FROM workspace_runnable_dependencies wru\n JOIN flow f\n ON wru.flow_path = f.path AND wru.workspace_id = f.workspace_id\n WHERE wru.runnable_path = $1 AND wru.runnable_is_agent AND wru.workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "46e65196c2a4f07d171a22f1e45c6ac927bd0e6b0f383626e673225065ee90bf" +} diff --git a/backend/.sqlx/query-a54e2334c365f90577f68ebefc8f3bee9b93ba0387f566ad7f502cbee818296e.json b/backend/.sqlx/query-a54e2334c365f90577f68ebefc8f3bee9b93ba0387f566ad7f502cbee818296e.json deleted file mode 100644 index 4e3b39403f..0000000000 --- a/backend/.sqlx/query-a54e2334c365f90577f68ebefc8f3bee9b93ba0387f566ad7f502cbee818296e.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO workspace_runnable_dependencies (flow_path, runnable_path, script_hash, runnable_is_flow, workspace_id, app_path)\n SELECT flow_path, runnable_path, script_hash, runnable_is_flow, $1, app_path\n FROM workspace_runnable_dependencies\n WHERE workspace_id = $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Text" - ] - }, - "nullable": [] - }, - "hash": "a54e2334c365f90577f68ebefc8f3bee9b93ba0387f566ad7f502cbee818296e" -} diff --git a/backend/.sqlx/query-fa8c36eda6d4cb64b4ac5979cc4eea76ab0226a514d25712a733e01add508b71.json b/backend/.sqlx/query-fa8c36eda6d4cb64b4ac5979cc4eea76ab0226a514d25712a733e01add508b71.json new file mode 100644 index 0000000000..6a4b3fe6de --- /dev/null +++ b/backend/.sqlx/query-fa8c36eda6d4cb64b4ac5979cc4eea76ab0226a514d25712a733e01add508b71.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace_runnable_dependencies (flow_path, runnable_path, runnable_is_flow, runnable_is_agent, workspace_id) VALUES ($1, $2, FALSE, TRUE, $3) ON CONFLICT DO NOTHING", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "fa8c36eda6d4cb64b4ac5979cc4eea76ab0226a514d25712a733e01add508b71" +} diff --git a/backend/migrations/20260915111928_record_linked_agents_as_runnable_dependencies.down.sql b/backend/migrations/20260915111928_record_linked_agents_as_runnable_dependencies.down.sql new file mode 100644 index 0000000000..c749fbe1b7 --- /dev/null +++ b/backend/migrations/20260915111928_record_linked_agents_as_runnable_dependencies.down.sql @@ -0,0 +1,9 @@ +DELETE FROM workspace_runnable_dependencies WHERE runnable_is_agent; + +DROP INDEX flow_workspace_without_hash_unique_idx; + +CREATE UNIQUE INDEX flow_workspace_without_hash_unique_idx + ON workspace_runnable_dependencies (flow_path, runnable_path, runnable_is_flow, workspace_id) + WHERE script_hash IS NULL; + +ALTER TABLE workspace_runnable_dependencies DROP COLUMN runnable_is_agent; diff --git a/backend/migrations/20260915111928_record_linked_agents_as_runnable_dependencies.up.sql b/backend/migrations/20260915111928_record_linked_agents_as_runnable_dependencies.up.sql new file mode 100644 index 0000000000..eab08e5c17 --- /dev/null +++ b/backend/migrations/20260915111928_record_linked_agents_as_runnable_dependencies.up.sql @@ -0,0 +1,21 @@ +-- A flow step linked to a saved agent (an `ai_agent` resource) is recorded next to the scripts and +-- subflows the flow runs, so renaming the agent can name the flows it would break. An agent row is +-- neither a script nor a flow: readers of script usages have to exclude it. +ALTER TABLE workspace_runnable_dependencies + ADD COLUMN runnable_is_agent BOOLEAN NOT NULL DEFAULT false; + +-- A script step and a linked agent can share a path. Without the flag in the key, the second +-- insert's ON CONFLICT DO NOTHING would silently drop one of the two rows. +DROP INDEX flow_workspace_without_hash_unique_idx; + +CREATE UNIQUE INDEX flow_workspace_without_hash_unique_idx + ON workspace_runnable_dependencies (flow_path, runnable_path, runnable_is_flow, runnable_is_agent, workspace_id) + WHERE script_hash IS NULL; + +-- The worker only records a flow when it is next deployed, so seed the ones already linking an +-- agent from their current value. +INSERT INTO workspace_runnable_dependencies (flow_path, runnable_path, runnable_is_flow, runnable_is_agent, workspace_id) +SELECT DISTINCT f.path, agent_ref #>> '{}', false, true, f.workspace_id +FROM flow f +CROSS JOIN LATERAL jsonb_path_query(f.value, 'lax $.** ? (@.type == "aiagent" && @.agent.type() == "string").agent') AS agent_ref +ON CONFLICT DO NOTHING; diff --git a/backend/summarized_schema.txt b/backend/summarized_schema.txt index b93a4bf6fc..1e27ee1ce4 100644 --- a/backend/summarized_schema.txt +++ b/backend/summarized_schema.txt @@ -232,7 +232,7 @@ workspace_key: workspace_id(char), kind(workspace_key_kind), key(char) FK: (workspace_id) -> workspace(id) workspace_protection_rule: workspace_id(char), name(char), rules(int), bypass_groups(text[]), bypass_users(text[]), created_at(ts) FK: (workspace_id) -> workspace(id) -workspace_runnable_dependencies: flow_path(char), runnable_path(char), script_hash(bigint), runnable_is_flow(bool), workspace_id(char), app_path(char), id(bigint) +workspace_runnable_dependencies: flow_path(char), runnable_path(char), script_hash(bigint), runnable_is_flow(bool), workspace_id(char), app_path(char), id(bigint), runnable_is_agent(bool) FK: (app_path, workspace_id) -> app(path, workspace_id) | (flow_path, workspace_id) -> flow(path, workspace_id) workspace_settings: workspace_id(char), slack_team_id(char), slack_name(char), slack_command_script(char), slack_email(char), customer_id(char), plan(char), webhook(text), ai_config(jsonb), large_file_storage(jsonb), git_sync(jsonb), default_app(char), default_scripts(jsonb), deploy_ui(jsonb), mute_critical_alerts(bool), color(char), operator_settings(jsonb), teams_command_script(text), teams_team_id(text), teams_team_name(text), git_app_installations(jsonb), ducklake(jsonb), slack_oauth_client_id(char), slack_oauth_client_secret(char), datatable(jsonb), teams_team_guid(text), auto_invite(jsonb), error_handler(jsonb), success_handler(jsonb), public_app_execution_limit_per_minute(int), dbt_warehouses(jsonb), guest_access_enabled(bool), guest_jwt_public_key(text), guest_jwt_jwks_url(text), ai_sessions_backup_generation(int) FK: (workspace_id) -> workspace(id) diff --git a/backend/windmill-api-flows/src/flows.rs b/backend/windmill-api-flows/src/flows.rs index 2093ac5fec..525e3cca46 100644 --- a/backend/windmill-api-flows/src/flows.rs +++ b/backend/windmill-api-flows/src/flows.rs @@ -77,6 +77,10 @@ pub fn workspaced_service() -> Router { "/list_paths_from_workspace_runnable/{runnable_kind}/{*path}", get(list_paths_from_workspace_runnable), ) + .route( + "/list_paths_linking_agent/{*path}", + get(list_paths_linking_agent), + ) .route("/history_update/v/{version}", post(update_flow_history)) .route("/get/v/{version}", get(get_flow_version_by_id)) .route("/get/v/{version}/p/{*path}", get(get_flow_version)) @@ -508,7 +512,7 @@ async fn list_paths_from_workspace_runnable( FROM workspace_runnable_dependencies wru JOIN flow f ON wru.flow_path = f.path AND wru.workspace_id = f.workspace_id - WHERE wru.runnable_path LIKE $1 || '%' AND wru.runnable_is_flow = $2 AND wru.workspace_id = $3"#, + WHERE wru.runnable_path LIKE $1 || '%' AND wru.runnable_is_flow = $2 AND NOT wru.runnable_is_agent AND wru.workspace_id = $3"#, path, matches!(runnable_kind, RunnableKind::Flow), w_id @@ -521,7 +525,7 @@ async fn list_paths_from_workspace_runnable( FROM workspace_runnable_dependencies wru JOIN flow f ON wru.flow_path = f.path AND wru.workspace_id = f.workspace_id - WHERE wru.runnable_path = $1 AND wru.runnable_is_flow = $2 AND wru.workspace_id = $3"#, + WHERE wru.runnable_path = $1 AND wru.runnable_is_flow = $2 AND NOT wru.runnable_is_agent AND wru.workspace_id = $3"#, path, matches!(runnable_kind, RunnableKind::Flow), w_id @@ -534,6 +538,30 @@ async fn list_paths_from_workspace_runnable( Ok(Json(runnables)) } +/// Flows with a step linked to the `ai_agent` resource at `path`, as of their last deploy. +async fn list_paths_linking_agent( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, +) -> JsonResult> { + let path = path.to_path(); + check_scopes(&authed, || format!("flows:read:agent/{}", path))?; + let mut tx = user_db.begin(&authed).await?; + let flows = sqlx::query_scalar!( + r#"SELECT DISTINCT f.path + FROM workspace_runnable_dependencies wru + JOIN flow f + ON wru.flow_path = f.path AND wru.workspace_id = f.workspace_id + WHERE wru.runnable_path = $1 AND wru.runnable_is_agent AND wru.workspace_id = $2"#, + path, + w_id + ) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + Ok(Json(flows)) +} + async fn validate_flow(new_flow: &NewFlow) -> error::Result<()> { #[cfg(not(feature = "enterprise"))] if new_flow.ws_error_handler_muted.is_some_and(|val| val) { diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 3330ed697f..3e48e7baca 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -7471,8 +7471,8 @@ async fn clone_workspace_runnable_dependencies( ) -> Result<()> { // Clone workspace_runnable_dependencies sqlx::query!( - "INSERT INTO workspace_runnable_dependencies (flow_path, runnable_path, script_hash, runnable_is_flow, workspace_id, app_path) - SELECT flow_path, runnable_path, script_hash, runnable_is_flow, $1, app_path + "INSERT INTO workspace_runnable_dependencies (flow_path, runnable_path, script_hash, runnable_is_flow, runnable_is_agent, workspace_id, app_path) + SELECT flow_path, runnable_path, script_hash, runnable_is_flow, runnable_is_agent, $1, app_path FROM workspace_runnable_dependencies WHERE workspace_id = $2", target_workspace_id, diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index c88a298646..8f188bf763 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -12103,6 +12103,25 @@ paths: items: type: string + /w/{workspace}/flows/list_paths_linking_agent/{path}: + get: + summary: list flow paths with a step linked to a saved agent + operationId: listFlowPathsLinkingAgent + tags: + - flow + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Path" + responses: + "200": + description: paths of the flows linking the `ai_agent` resource, as of their last deploy + content: + application/json: + schema: + type: array + items: + type: string + /w/{workspace}/flows/get/v/{version}: get: summary: get flow version diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index 1f3b9f1d56..03150beb00 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -1795,6 +1795,17 @@ async fn lock_modules( agent, tool_inputs, } => { + if let Some(agent_path) = agent.as_deref().filter(|_| !skip_flow_update) { + sqlx::query!( + "INSERT INTO workspace_runnable_dependencies (flow_path, runnable_path, runnable_is_flow, runnable_is_agent, workspace_id) VALUES ($1, $2, FALSE, TRUE, $3) ON CONFLICT DO NOTHING", + job_path, + agent_path, + job.workspace_id, + ) + .execute(db) + .await?; + } + // Extract FlowModules from tools and track their original indices // MCP tools don't need locking, so we filter them out let mut flow_modules = Vec::new(); diff --git a/frontend/src/lib/components/FlowBuilder.svelte b/frontend/src/lib/components/FlowBuilder.svelte index 6888c61f2d..97b871921c 100644 --- a/frontend/src/lib/components/FlowBuilder.svelte +++ b/frontend/src/lib/components/FlowBuilder.svelte @@ -510,8 +510,8 @@ draftAgents.map((a) => [a.path, agentDraftCanWrite(a, user ?? $userStore ?? undefined)]) ) // The path is passed, so a draft that renames the agent is refused here too: a rename is - // the resource editor's to deploy, and this dialog lists the agent under the path the - // flow links. + // the agent editor's to deploy, and this dialog lists the agent under the path the flow + // links. agentRefusal = Object.fromEntries( draftAgents.map((a) => [a.path, agentDraftDeployRefusal(a.state, a.path)]) ) diff --git a/frontend/src/lib/components/Path.svelte b/frontend/src/lib/components/Path.svelte index b45d980246..ee87cc9511 100644 --- a/frontend/src/lib/components/Path.svelte +++ b/frontend/src/lib/components/Path.svelte @@ -30,6 +30,7 @@ import { createEventDispatcher, getContext, untrack } from 'svelte' import { writable } from 'svelte/store' import { Alert, Button } from './common' + import { overlayStack, type OverlayStack } from './common/overlayHost.svelte' import { random_adj } from './random_positive_adjetive' import { ChevronDown, Copy, SearchCode } from 'lucide-svelte' import Tooltip from './Tooltip.svelte' @@ -405,9 +406,11 @@ } }) - const openSearchWithPrefilledText: (t?: string) => void = getContext( + const openSearchWithPrefilledText: (t?: string, stack?: OverlayStack) => void = getContext( 'openSearchWithPrefilledText' ) + // Handed to the search so it stacks above the modal or drawer this field sits in. + const searchStack = overlayStack() $effect.pre(() => { ;[meta?.name, meta?.owner, meta?.ownerKind] @@ -451,14 +454,18 @@ initialPath !== path ) let pathUsageInFlowsPromise = $derived( - (kind == 'script' || kind == 'flow') && - ws && - initialPath && - FlowService.listFlowPathsFromWorkspaceRunnable({ - workspace: ws, - path: initialPath, - runnableKind: kind - }) + ws && initialPath + ? kind == 'script' || kind == 'flow' + ? FlowService.listFlowPathsFromWorkspaceRunnable({ + workspace: ws, + path: initialPath, + runnableKind: kind + }) + : kind == 'resource' + ? // Only steps linked to a saved agent are tracked; other `$res:` references are not. + FlowService.listFlowPathsLinkingAgent({ workspace: ws, path: initialPath }) + : undefined + : undefined ) let pathUsageInAppsPromise = $derived( (kind == 'script' || kind == 'flow') && @@ -647,24 +654,36 @@ {/if} + {:else if displayPathChangedWarning && kind == 'resource'} + {@render renameMayBreakWarning()} + {/if} + {:catch} + + {#if displayPathChangedWarning && kind == 'resource'} + {@render renameMayBreakWarning()} {/if} {/await} {:else if displayPathChangedWarning} - - You are renaming an item that may be depended upon by other items. This may break apps, flows - or resources. Find if it used elsewhere using the content search. Note that linked variables - and resources (having the same path) are automatically moved together. -
    - -
    -     + {@render renameMayBreakWarning()} {/if}
    + +{#snippet renameMayBreakWarning()} + + You are renaming an item that may be depended upon by other items. This may break apps, flows or + resources. Find if it used elsewhere using the content search. Note that linked variables and + resources (having the same path) are automatically moved together. +
    + +
    +     +{/snippet} diff --git a/frontend/src/lib/components/flows/FlowEditor.svelte b/frontend/src/lib/components/flows/FlowEditor.svelte index 73482817ec..bf25a1dcda 100644 --- a/frontend/src/lib/components/flows/FlowEditor.svelte +++ b/frontend/src/lib/components/flows/FlowEditor.svelte @@ -4,6 +4,7 @@ import FlowEditorPanel from './content/FlowEditorPanel.svelte' import { agentEditorTarget, type AgentEditorTarget } from './agentEditorStore.svelte' import AgentEditorModal from './content/AgentEditorModal.svelte' + import { repointLinkedAgent } from './linkedAgentDrafts' import FlowModuleSchemaMap from './map/FlowModuleSchemaMap.svelte' import type { OpenInSessionSource } from '$lib/components/sessions/OpenInSessionButton.svelte' import WindmillIcon from '../icons/WindmillIcon.svelte' @@ -551,4 +552,5 @@ t.host?.flowPath === $pathStore && targetWorkspace(t) === editorWorkspace} + onRenamed={(from, to) => repointLinkedAgent(flowStore.val.value, from, to)} /> diff --git a/frontend/src/lib/components/flows/agentDraft.svelte.ts b/frontend/src/lib/components/flows/agentDraft.svelte.ts index d4f8fa35de..5ee65cd98a 100644 --- a/frontend/src/lib/components/flows/agentDraft.svelte.ts +++ b/frontend/src/lib/components/flows/agentDraft.svelte.ts @@ -112,12 +112,11 @@ export function agentDraftDeployRefusal( if (blocked) { return blocked } - // Renaming is not the agent editor's to do: moving the resource leaves every step that links to - // it naming a path that no longer exists, and reconciling those is a feature of its own. A - // renamed path can still reach here, the generic editor writing the same draft row and offering - // a path field, so refuse it rather than performing half of a rename. + // A rename is the agent editor's to deploy: it repoints the steps of the flow it was opened from. + // Deployed from anywhere that names the path it writes to (a flow's deploy dialog, which lists the + // agent under the path the flow links), it would move the agent out from under that flow. if (currentPath && state.path !== currentPath) { - return `This draft renames the agent to ${state.path}. Deploy it from the resource editor instead.` + return `This draft renames the agent to ${state.path}. Deploy it from the agent editor instead.` } // Only a draft naming another type: the load refuses a resource that is not an agent, while a // draft the generic resource editor wrote names no type at all and inherits the loaded one. @@ -132,6 +131,9 @@ export function agentDraftDeployRefusal( * persisted draft row: the form stays editable while a deploy is in flight. Surfaces that deploy * the row itself go through `deployDraft` instead. * + * `fromPath` is the path the editor loaded; `state.path` differs from it when the draft renames the + * agent, and the update then moves the resource there. + * * `notAnAgent` separates the one failure that invalidates the caller's whole view of the path, its * holding something else now, from a write that merely failed. */ @@ -139,6 +141,7 @@ type AgentWriteResult = { ok: true } | { ok: false; error: string; notAnAgent?: async function writeAgentResource( workspace: string, + fromPath: string, state: AgentResourceState, noDeployed: boolean ): Promise { @@ -161,12 +164,12 @@ async function writeAgentResource( // its own: were the path deleted and recreated as something else meanwhile, this write // would put an agent config inside that resource. Reading it again narrows the window to // the request rather than to however long the editor or the dialog stayed open. - const current = await ResourceService.getResource({ workspace, path: state.path }) - const refused = agentEditorRefusal(state.path, current.resource_type) + const current = await ResourceService.getResource({ workspace, path: fromPath }) + const refused = agentEditorRefusal(fromPath, current.resource_type) if (refused) { return { ok: false, error: refused, notAnAgent: true } } - await ResourceService.updateResource({ workspace, path: state.path, requestBody: body }) + await ResourceService.updateResource({ workspace, path: fromPath, requestBody: body }) } } catch (err) { return { ok: false, error: `Could not save agent: ${err}` } @@ -193,8 +196,9 @@ export interface AgentDraftHandle { /** Why this path cannot be edited here, if it cannot. Render it instead of the form. */ readonly refusal: string | undefined readonly sync: TriggerDraftSync - /** Write the current state to the resource and drop the draft. */ - deploy: () => Promise + /** Write the current state to the resource and drop the draft. Resolves to the path written, + * which differs from the one loaded when the draft renames the agent, or undefined on failure. */ + deploy: () => Promise } /** @@ -333,21 +337,23 @@ export function useAgentDraft(opts: AgentDraftOptions): AgentDraftHandle { }) }) - async function deploy(): Promise { + async function deploy(): Promise { const ws = opts.workspace() + const fromPath = opts.path() const s = state - if (!ws || !s) return false - const refused = agentDraftDeployRefusal(s, opts.path()) + if (!ws || !fromPath || !s) return undefined + // No path to hold the draft to: renaming is this editor's to deploy. + const refused = agentDraftDeployRefusal(s, undefined) if (refused) { sendUserToast(refused, true) - return false + return undefined } // The form stays editable while the request is in flight, so everything below works from a // snapshot taken now. Adopting the live state as `deployed` afterwards would count an edit // made during the request as saved, and the banner would clear on a value the server never // received; against the snapshot it stays a draft, which is what it is. const submitted = structuredClone($state.snapshot(s)) as AgentResourceState - const written = await writeAgentResource(ws, submitted, noDeployed) + const written = await writeAgentResource(ws, fromPath, submitted, noDeployed) if (!written.ok) { // A path that is no longer an agent tears this editor down; anything else is a plain error // the user can retry from the form as it stands. @@ -356,29 +362,31 @@ export function useAgentDraft(opts: AgentDraftOptions): AgentDraftHandle { } else { sendUserToast(written.error, true) } - return false + return undefined } // The counter the step card's write-back used to report, from the surface that now owns the // write: a deploy here reaches every flow linking this agent. logReusableAgentUsage(noDeployed ? 'saved' : 'updated') deployed = submitted noDeployed = false + const renamed = submitted.path !== fromPath // Only when the form still holds exactly what was sent. `discard` resets the handle's cell to // what it is given, and the apply-effect copies that back over the form: against an edit made // while the request was in flight that would erase it, draft and all. Such an edit is a real - // unsaved change over the version just deployed, so it keeps its draft and its banner. - if (!deepEqual($state.snapshot(state), submitted)) { + // unsaved change over the version just deployed, so it keeps its draft and its banner. Not + // after a rename: the draft is keyed on a path that no longer names the agent. + if (!renamed && !deepEqual($state.snapshot(state), submitted)) { sendUserToast(`Saved agent ${submitted.path}. Later edits are still unsaved`) loadedFor = `${ws}:${submitted.path}` - return true + return submitted.path } // `discard`, not `remove`: it resets the handle's cell to what was just saved, so the // apply-effect cannot bounce the form back to the now-stale draft. - sync.discard(opts.path()!, submitted) + sync.discard(fromPath, submitted) // A rename moves the row, so the next load must not reuse the old key. loadedFor = `${ws}:${submitted.path}` - sendUserToast(`Saved agent ${submitted.path}`) - return true + sendUserToast(renamed ? `Renamed agent to ${submitted.path}` : `Saved agent ${submitted.path}`) + return submitted.path } return { diff --git a/frontend/src/lib/components/flows/content/AgentEditorHost.svelte b/frontend/src/lib/components/flows/content/AgentEditorHost.svelte index 9fd562692c..b6a98160be 100644 --- a/frontend/src/lib/components/flows/content/AgentEditorHost.svelte +++ b/frontend/src/lib/components/flows/content/AgentEditorHost.svelte @@ -33,6 +33,9 @@ import { AGENT_EDITOR_RUN_INPUTS, AGENT_TOOLS_ROW } from '../agentFormFields' import { toolDisplayName, type AgentTool } from '../agentToolUtils' import { useAgentDraft } from '../agentDraft.svelte' + import Path from '$lib/components/Path.svelte' + import Label from '$lib/components/Label.svelte' + import { sendUserToast } from '$lib/toast' interface Props { /** The `ai_agent` resource being edited. */ @@ -320,13 +323,19 @@ if (toolId === id) onSelectTool?.(undefined) } + /** The path field's own verdict (a taken path, an invalid name), which the server would otherwise + * only report after the request. */ + let pathError = $state('') + export function deploy(): Promise { - return draft.deploy().then(async (ok) => { - // The path this editor opened, not the draft's live one: `deploy` refuses a renaming draft, - // so the write always lands here, while the shared draft can be repointed by another tab - // mid-request and would send the reconciliation after a resource nobody wrote. - if (ok) await onSaved?.(path) - return ok + if (pathError) { + sendUserToast(`Cannot deploy the agent: ${pathError}`, true) + return Promise.resolve(false) + } + return draft.deploy().then(async (written) => { + // The path the write landed on, which a rename moves off the one this editor opened. + if (written) await onSaved?.(written) + return written !== undefined }) } export function draftHandle() { @@ -352,6 +361,25 @@
    +
    + +
    boolean + /** A deploy moved the agent from `from` to `to`. What names the old path belongs to the surface + * that opened the editor: a flow's own steps, a page's URL. */ + onRenamed?: (from: string, to: string) => void } - let { enableAi = false, owns }: Props = $props() + let { enableAi = false, owns, onRenamed = undefined }: Props = $props() // Every target names the surface that opened it, and only a flow step or a resource row can: // an agent used as a tool of the agent being edited stays part of it, with no way in this editor @@ -178,9 +182,10 @@ if (!at.host) return // The host graph resolves a linked agent's tool nodes from the resource, so it has to re-read // what the write just changed. Every step of that flow linking this agent, not only the one - // the editor was opened from: they all show tools the write may have moved. + // the editor was opened from: they all show tools the write may have moved. Looked up under + // the path the editor opened, since a rename is about to move those steps off it. const scope = linkedToolsScope(at.ws, at.host.flowPath) - const moduleIds = new Set(linkedModulesForAgent(scope, path)) + const moduleIds = new Set(linkedModulesForAgent(scope, at.path)) moduleIds.add(at.host.moduleId) return Promise.all( // With the draft: a deploy leaves none, but a version restore leaves the draft standing and @@ -202,10 +207,21 @@ } } - /** What a successful deploy has to reconcile. The path is the one it wrote, which `deploy` holds - * to the one the editor opened: this editor does not rename. */ + /** What a successful deploy has to reconcile. `savedPath` is the path it wrote, which a rename + * moves off the one the editor opened. */ async function onSaved(savedPath: string) { - await reconcile(deployingFor ?? currentWriteTarget(), savedPath) + const at = deployingFor ?? currentWriteTarget() + // Before the rename is announced: it finds the steps to refresh under the old path. + const reconciled = reconcile(at, savedPath) + if (at && savedPath !== at.path) { + onRenamed?.(at.path, savedPath) + // The dialog is keyed on the path, so this reloads it on the renamed agent. Only while it + // still shows the one deployed: it can be closed or pointed elsewhere mid-request. + if (target?.path === at.path) { + openAgentEditor({ path: savedPath, workspace: target.workspace, host: target.host }) + } + } + await reconciled } diff --git a/frontend/src/lib/components/flows/content/AgentResourceBar.svelte b/frontend/src/lib/components/flows/content/AgentResourceBar.svelte index 96548b10af..fafe69ebb4 100644 --- a/frontend/src/lib/components/flows/content/AgentResourceBar.svelte +++ b/frontend/src/lib/components/flows/content/AgentResourceBar.svelte @@ -33,7 +33,11 @@ } from '../linkedAgentToolsStore.svelte' import { logReusableAgentUsage } from '../agentTelemetry' import { claimLinkedToolsFetch } from '../flowState' - import { AgentDraftUnavailable, fetchAgentWithDraft } from '../linkedAgentDrafts' + import { + AgentDraftUnavailable, + fetchAgentWithDraft, + isExpectedLinkFailure + } from '../linkedAgentDrafts' import type { AgentResourceState } from '../agentDraft.svelte' import { getLocalDraftHint } from '$lib/localDraftHints.svelte' import Tooltip from '$lib/components/meltComponents/Tooltip.svelte' @@ -103,6 +107,30 @@ fromDraft: boolean providerPath?: string providerOk: boolean + /** The link cannot be read. `missing` (404): nothing exists at the path, the agent having been + * renamed or deleted. `forbidden` (401/403): it exists and this user is refused it, a folder + * they cannot read included, which says nothing about whether a run of the flow can read it. + * Returned rather than thrown so it is guarded like any result. */ + unavailable?: 'missing' | 'forbidden' + } + + async function fetchLinkedAgent( + path: string, + ws: string + ): Promise<{ response: Resource; draft: AgentResourceState | undefined }> { + try { + return await fetchAgentWithDraft(path, ws) + } catch (err) { + // Only the DRAFT was unreadable. This card is a display, so fall back to the deployed + // agent rather than rendering one with no brain and no tools, which reads as "the agent + // is empty" while the Draft badge still says it has unsaved changes. Same fallback the + // graph's tool nodes take; the paths that run or deploy the draft still refuse. + if (!(err instanceof AgentDraftUnavailable)) throw err + return { + response: await ResourceService.getResource({ workspace: ws, path }), + draft: undefined + } + } } // A linked agent is rigid and read-only: its brain and tools come from the resource. We @@ -112,29 +140,27 @@ let linkedResource = resource( () => ({ ws, path: agent, writes, draftSaves }), async ({ ws, path, writes, draftSaves }): Promise => { + const empty = { + ws, + path, + writes, + draftSaves, + config: {}, + tools: [], + fromDraft: false, + providerOk: true + } if (!ws || !path) { - return { - ws, - path, - writes, - draftSaves, - config: {}, - tools: [], - fromDraft: false, - providerOk: true - } + return empty } let response: Resource let draft: AgentResourceState | undefined try { - ;({ response, draft } = await fetchAgentWithDraft(path, ws)) + ;({ response, draft } = await fetchLinkedAgent(path, ws)) } catch (err) { - // Only the DRAFT was unreadable. This card is a display, so fall back to the deployed - // agent rather than rendering one with no brain and no tools, which reads as "the agent - // is empty" while the Draft badge still says it has unsaved changes. Same fallback the - // graph's tool nodes take; the paths that run or deploy the draft still refuse. - if (!(err instanceof AgentDraftUnavailable)) throw err - response = await ResourceService.getResource({ workspace: ws, path }) + if (!isExpectedLinkFailure(err)) throw err + const status = (err as { status?: number }).status + return { ...empty, unavailable: status === 404 ? 'missing' : 'forbidden' } } const cfg = (draft?.args ?? response.value ?? {}) as AIAgentConfig & { provider?: { resource?: string } @@ -189,6 +215,7 @@ let brainParams = $derived(summarizeAgentBrain(linkedInfo?.config)) let providerPath = $derived(linkedInfo?.providerPath) let providerOk = $derived(linkedInfo?.providerOk ?? true) + let unavailable = $derived(linkedInfo?.unavailable ?? false) // The hint flips on the first keystroke in the agent editor, so the badge does not wait for the // debounced autosave and the refetch behind it; the fetched answer covers a draft written // elsewhere, which no editor here has published an opinion about. @@ -468,6 +495,15 @@ } } + // A link naming nothing readable has nothing to fork. Dropping it leaves a standalone step with its + // flow-local inputs, to configure here or replace with a saved agent; the tool overrides were + // keyed by the missing agent's tools, so they go with it. + function removeLink() { + toolInputs = {} + agent = undefined + sendUserToast('Removed the link to the missing agent') + } + // Edit the saved agent itself. The step stays linked throughout: the edits live in the agent's // own resource draft, not in this step, so they survive leaving the flow and are the same edits // whichever flow — or the resources page — opened them. @@ -534,7 +570,7 @@ {/if} {/if} - {#if !fromAgentEditor} + {#if !fromAgentEditor && !unavailable}
    {#if showDetail && (brainParams.length > 0 || inheritedTools.length > 0)} @@ -581,7 +619,32 @@ {/if} - {#if !providerOk} + {#if unavailable === 'forbidden'} +
    + + You don't have access to {agent}, so its configuration + can't be shown or edited here. + +
    + {:else if unavailable === 'missing'} +
    + + No saved agent exists at {agent}. It may have been + renamed or deleted. Remove the link to configure the step here, or add the agent again + from Saved agents. +
    + +
    +     +
    + {:else if !providerOk}
    This agent's model provider{#if providerPath} diff --git a/frontend/src/lib/components/flows/linkedAgentDrafts.test.ts b/frontend/src/lib/components/flows/linkedAgentDrafts.test.ts index 563007615f..fa72aaba5c 100644 --- a/frontend/src/lib/components/flows/linkedAgentDrafts.test.ts +++ b/frontend/src/lib/components/flows/linkedAgentDrafts.test.ts @@ -4,6 +4,7 @@ import { inlineAgentDraft, inlineAgentDrafts, loadLinkedAgentDrafts, + repointLinkedAgent, type LinkedAgentDraft } from './linkedAgentDrafts' import { ResourceService, type FlowModule, type FlowValue } from '$lib/gen' @@ -118,6 +119,38 @@ describe('inlineAgentDrafts', () => { }) }) +describe('repointLinkedAgent', () => { + // A rename from the agent editor moves every step of the host flow onto the new path, nested ones + // included. Miss one and it silently stays linked to a path that no longer exists. + it('repoints linked steps at any depth and leaves other agents alone', () => { + const value = { + modules: [ + { id: 'a', value: { type: 'aiagent', agent: 'f/team/support', tools: [] } }, + { + id: 'b', + value: { + type: 'branchall', + branches: [ + { + modules: [ + { id: 'c', value: { type: 'aiagent', agent: 'f/team/support', tools: [] } }, + { id: 'd', value: { type: 'aiagent', agent: 'f/team/other', tools: [] } } + ] + } + ] + } + } + ] + } as unknown as FlowValue + + expect(repointLinkedAgent(value, 'f/team/support', 'f/team/helpdesk')).toEqual(['a', 'c']) + const branch = (value.modules[1].value as any).branches[0].modules + expect((value.modules[0].value as any).agent).toBe('f/team/helpdesk') + expect(branch[0].value.agent).toBe('f/team/helpdesk') + expect(branch[1].value.agent).toBe('f/team/other') + }) +}) + // A link the user cannot resolve is an ordinary state and must not block the flow; anything else is // an outage, and answering "no draft" to one would silently test or deploy against the deployed // agent while the editor shows the draft. diff --git a/frontend/src/lib/components/flows/linkedAgentDrafts.ts b/frontend/src/lib/components/flows/linkedAgentDrafts.ts index 8da732e7ff..65624d4429 100644 --- a/frontend/src/lib/components/flows/linkedAgentDrafts.ts +++ b/frontend/src/lib/components/flows/linkedAgentDrafts.ts @@ -35,6 +35,25 @@ export function linkedAgentPaths(value: FlowValue | undefined): string[] { return [...paths] } +/** Point every step of this flow linked to `from` at `to`, for an agent renamed from inside it. + * Returns the ids of the steps it moved. */ +export function repointLinkedAgent( + value: FlowValue | undefined, + from: string, + to: string +): string[] { + if (!value?.modules) return [] + const moved: string[] = [] + for (const module of dfs(value.modules, (m) => m)) { + const v = module?.value as { type?: string; agent?: string } | undefined + if (v?.type === 'aiagent' && v.agent === from) { + v.agent = to + moved.push(module.id) + } + } + return moved +} + /** * The unsaved draft for an agent, freshest first: the cell an open agent editor is writing, then * what a `get_draft` response carried. @@ -114,7 +133,7 @@ export function agentDraftCanWrite(draft: LinkedAgentDraft, user: UserExt | unde * neither should stop the caller — the flow still tests and deploys, against the deployed agent. * Every other failure is an outage, and answering "no draft" to one would quietly run or deploy * the wrong configuration, which is the whole thing this module exists to prevent. */ -function isExpectedLinkFailure(err: unknown): boolean { +export function isExpectedLinkFailure(err: unknown): boolean { const status = (err as { status?: number } | null | undefined)?.status return status === 401 || status === 403 || status === 404 } diff --git a/frontend/src/lib/components/search/GlobalSearchModal.svelte b/frontend/src/lib/components/search/GlobalSearchModal.svelte index 084ad9d492..b94e0faefa 100644 --- a/frontend/src/lib/components/search/GlobalSearchModal.svelte +++ b/frontend/src/lib/components/search/GlobalSearchModal.svelte @@ -29,6 +29,8 @@ WandSparkles } from 'lucide-svelte' import Portal from '$lib/components/Portal.svelte' + import { zIndexes } from '$lib/zIndexes' + import { overlayStack } from '$lib/components/common/overlayHost.svelte' import { twMerge } from 'tailwind-merge' import ContentSearchInner from '../ContentSearchInner.svelte' @@ -367,6 +369,7 @@ async function handleKeydown(event: KeyboardEvent) { if ((!isMac() ? event.ctrlKey : event.metaKey) && event.key === 'k') { event.preventDefault() + if (!open) openedOn = undefined await openModal() } if (open) { @@ -450,6 +453,24 @@ mouseMoved = true } + // On the overlay stack while open: a modal or drawer it was opened from arbitrates Escape on that + // stack, and would otherwise close itself on the key meant for the search above it. The opener + // names its stack, because a pane hosting an editor (a sessions tab) keeps its own. + const globalStack = overlayStack() + let openedOn: import('$lib/components/common/overlayHost.svelte').OverlayStack | undefined = + $state(undefined) + const STACK_ID = 'global-search' + $effect(() => { + if (!open) return + const stack = openedOn ?? globalStack + untrack(() => stack.val.push(STACK_ID)) + return () => { + untrack(() => { + stack.val = stack.val.filter((id) => id !== STACK_ID) + }) + } + }) + onMount(() => { window.addEventListener('keydown', handleKeydown) window.addEventListener('mousemove', handleMouseMove) @@ -559,7 +580,11 @@ } } - export async function openSearchWithPrefilledText(text?: string) { + export async function openSearchWithPrefilledText( + text?: string, + stack?: import('$lib/components/common/overlayHost.svelte').OverlayStack + ) { + openedOn = stack await openModal() searchTerm = text ?? searchTerm await handleSearch() @@ -618,9 +643,9 @@
    {#if agentEditorTarget()} {#await import('$lib/components/flows/content/AgentEditorModal.svelte') then { default: AgentEditorModal }} - t.host === undefined} /> + t.host === undefined} + onRenamed={(from, to) => { + void loadResources() + // Only while the dialog still shows the agent: closed mid-request, it already cleared the + // anchor, and writing it back would reopen the editor on refresh. + if (agentEditorTarget()?.path !== from) return + // Claimed first, as a row click does, so the deep-link effect does not reopen it. + handledHash = `#/resource/${to}` + setPageDrawerAnchor(RESOURCES_PATH, to) + }} + /> {/await} {/if} From 73dc892f9c9c840a5f0fb12fcbb28bbe0f38795f Mon Sep 17 00:00:00 2001 From: hugocasa Date: Wed, 16 Sep 2026 10:45:34 +0200 Subject: [PATCH 142/144] fix: walk the whole fork ancestry for app installations and fork conflicts (#11151) * fix: walk the whole fork ancestry for app installations and fork conflicts Co-Authored-By: Claude Fable 5.1 * docs: describe the fork-conflict gate as ancestor-wide Co-Authored-By: Claude Fable 5.1 * chore: update ee-repo-ref to d252afcc80e77fcc4f9a2a346b80908c8605a6c0 This commit updates the EE repository reference after PR #803 was merged in windmill-ee-private. Previous ee-repo-ref: 5f68c8c351ffc92feccffe69a857b60be376464e New ee-repo-ref: d252afcc80e77fcc4f9a2a346b80908c8605a6c0 Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Fable 5.1 Co-authored-by: windmill-internal-app[bot] --- backend/ee-repo-ref.txt | 2 +- .../tests/fixtures/schedule_fork_conflict.sql | 46 ++++++++ backend/tests/git_sync_fork_credential.rs | 100 +++++++++++++++++- backend/tests/schedule_fork_conflict.rs | 59 +++++++++++ backend/windmill-api-schedule/src/lib.rs | 46 +++----- backend/windmill-common/src/workspaces.rs | 42 ++++++++ backend/windmill-trigger/src/handler.rs | 76 ++++--------- .../lib/components/ForkConflictModal.svelte | 16 +-- frontend/src/lib/stores.ts | 2 +- frontend/src/lib/utils/forkConflict.ts | 15 +-- 10 files changed, 301 insertions(+), 103 deletions(-) create mode 100644 backend/tests/fixtures/schedule_fork_conflict.sql create mode 100644 backend/tests/schedule_fork_conflict.rs diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 5d28f73b5a..317ca12c5e 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -93433d7c9dc34f2c0f56a5297d3453aabd2f9472 \ No newline at end of file +d252afcc80e77fcc4f9a2a346b80908c8605a6c0 diff --git a/backend/tests/fixtures/schedule_fork_conflict.sql b/backend/tests/fixtures/schedule_fork_conflict.sql new file mode 100644 index 0000000000..8e81d9ede6 --- /dev/null +++ b/backend/tests/fixtures/schedule_fork_conflict.sql @@ -0,0 +1,46 @@ +-- A three-deep fork chain whose schedule rows were cloned down at fork time. +-- The middle fork has since deleted its copy, so the leaf's schedule shares +-- its cron only with the root — the shape a direct-parent check misses. + +INSERT INTO workspace (id, name, owner, parent_workspace_id) VALUES + ('sfc-root', 'sfc-root', 'sfc-admin', NULL), + ('sfc-mid', 'sfc-mid', 'sfc-admin', 'sfc-root'), + ('sfc-leaf', 'sfc-leaf', 'sfc-admin', 'sfc-mid'); + +INSERT INTO workspace_key (workspace_id, kind, key) VALUES + ('sfc-root', 'cloud', 'sfc-root-key'), + ('sfc-mid', 'cloud', 'sfc-mid-key'), + ('sfc-leaf', 'cloud', 'sfc-leaf-key'); + +INSERT INTO workspace_settings (workspace_id) VALUES + ('sfc-root'), ('sfc-mid'), ('sfc-leaf'); + +INSERT INTO group_ (workspace_id, name, summary, extra_perms) VALUES + ('sfc-root', 'all', 'All users', '{}'), + ('sfc-mid', 'all', 'All users', '{}'), + ('sfc-leaf', 'all', 'All users', '{}'); + +INSERT INTO password(email, password_hash, login_type, super_admin, verified, name, username) + VALUES ('sfc-admin@windmill.dev', 'x', 'password', true, true, 'SFC Admin', 'sfc-admin'); + +INSERT INTO usr(workspace_id, email, username, is_admin, role) VALUES + ('sfc-root', 'sfc-admin@windmill.dev', 'sfc-admin', true, 'Admin'), + ('sfc-mid', 'sfc-admin@windmill.dev', 'sfc-admin', true, 'Admin'), + ('sfc-leaf', 'sfc-admin@windmill.dev', 'sfc-admin', true, 'Admin'); + +INSERT INTO token(token_hash, token_prefix, token, email, label, super_admin) + VALUES (encode(sha256('SFC_ADMIN_TOKEN'::bytea), 'hex'), 'SFC_ADMIN_', 'SFC_ADMIN_TOKEN', 'sfc-admin@windmill.dev', 't', true); + +-- Enabling pushes the next run, which needs the scheduled script to exist. +INSERT INTO script (workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES + ('sfc-leaf', 'sfc-admin', 'export async function main() { return "ok" }', '{}', '', '', 'f/shared/job', 7788001, 'deno', ''); + +INSERT INTO schedule (workspace_id, path, edited_by, edited_at, schedule, enabled, script_path, args, is_flow, email, timezone, extra_perms, permissioned_as) +VALUES + ('sfc-root', 'f/shared/nightly', 'sfc-admin', NOW(), '0 0 0 * * *', true, 'f/shared/job', '{}', false, 'sfc-admin@windmill.dev', 'UTC', '{}', 'u/sfc-admin'), + ('sfc-leaf', 'f/shared/nightly', 'sfc-admin', NOW(), '0 0 0 * * *', false, 'f/shared/job', '{}', false, 'sfc-admin@windmill.dev', 'UTC', '{}', 'u/sfc-admin'), + -- A path only the leaf has: nothing above shares it. + ('sfc-leaf', 'f/shared/own', 'sfc-admin', NOW(), '0 0 0 * * *', false, 'f/shared/job', '{}', false, 'sfc-admin@windmill.dev', 'UTC', '{}', 'u/sfc-admin'); + +GRANT ALL PRIVILEGES ON TABLE workspace_key TO windmill_admin; +GRANT ALL PRIVILEGES ON TABLE workspace_key TO windmill_user; diff --git a/backend/tests/git_sync_fork_credential.rs b/backend/tests/git_sync_fork_credential.rs index 10cf1e3735..9d5eb62dfb 100644 --- a/backend/tests/git_sync_fork_credential.rs +++ b/backend/tests/git_sync_fork_credential.rs @@ -12,8 +12,8 @@ use sqlx::{Pool, Postgres}; use windmill_common::git_sync_ee::{ - create_repo_webhook, git_credential_for_url, repo_provider, repo_supports_managed_git_features, - set_git_credential, GitProvider, + create_repo_webhook, git_app_installations_for, git_credential_for_url, managed_pr_base_branch, + repo_provider, repo_supports_managed_git_features, set_git_credential, GitProvider, }; use windmill_common::workspaces::GitCredentialProvider; @@ -278,3 +278,99 @@ async fn an_unreachable_gitlab_host_is_the_reported_error( ); Ok(()) } + +/// GitHub App installations are normally copied into a fork, but a workspace +/// attached as a dev workspace, or forked before its parent connected the App, +/// holds none, and neither does anything forked from it. The lookup reaches the +/// nearest workspace up the chain that holds some, and the background App path +/// (PR base resolution here) authenticates with that installation's token. +#[sqlx::test(fixtures("git_sync_fork_credential"))] +async fn app_installations_come_from_the_nearest_ancestor_holding_some( + db: Pool, +) -> anyhow::Result<()> { + use axum::{routing::get, Router}; + use std::sync::{Arc, Mutex}; + + // A stand-in GitHub API: one repository, and a record of who asked for it. + let seen: Arc>> = Arc::new(Mutex::new(vec![])); + let app = Router::new().route( + "/api/v3/repos/acme/repo", + get({ + let seen = seen.clone(); + move |headers: axum::http::HeaderMap| { + let seen = seen.clone(); + async move { + let auth = headers + .get("authorization") + .and_then(|v| v.to_str().ok()) + .unwrap_or("") + .to_string(); + seen.lock().unwrap().push(auth); + axum::Json(serde_json::json!({ "default_branch": "trunk" })) + } + } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let port = listener.local_addr()?.port(); + tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + let stub = format!("http://127.0.0.1:{port}"); + + // The root holds the installation, with a cached token so nothing is minted. + sqlx::query( + "UPDATE workspace_settings SET git_app_installations = $1::jsonb WHERE workspace_id = 'parent-ws'", + ) + .bind(serde_json::json!([{ + "installation_id": 42, "account_id": "acme", "jwt_token": "x", + "github_base_url": stub, + "installation_token": "root-token", "installation_token_expiration": 4102444800i64 + }])) + .execute(&db) + .await?; + // The fork's copy of the resource names the App-backed repository. + sqlx::query("UPDATE resource SET value = $1::jsonb WHERE workspace_id = 'deep-fork-ws' AND path = 'u/admin/repo'") + .bind(serde_json::json!({ "url": format!("{stub}/acme/repo.git"), "is_github_app": true })) + .execute(&db) + .await?; + + assert_eq!( + git_app_installations_for(&db, "deep-fork-ws").await?, + ("parent-ws".to_string(), vec![(42, Some(stub.clone()))]), + "two levels down, the root's installations are the ones to use" + ); + assert_eq!( + git_app_installations_for(&db, "orphan-ws").await?, + ("orphan-ws".to_string(), vec![]), + "a workspace with nothing above it resolves nothing" + ); + assert_eq!( + managed_pr_base_branch(&db, "deep-fork-ws", REPO) + .await? + .as_deref(), + Some("trunk"), + "the background App path reaches the repository through the root's installation" + ); + let seen = seen.lock().unwrap().clone(); + assert!( + !seen.is_empty() && seen.iter().all(|auth| auth == "Bearer root-token"), + "every call authenticated with the root's cached token: {seen:?}" + ); + + // A closer holder takes precedence over the root. + sqlx::query( + "UPDATE workspace_settings SET git_app_installations = $1::jsonb WHERE workspace_id = 'fork-ws'", + ) + .bind(serde_json::json!([{ + "installation_id": 7, "account_id": "acme", "jwt_token": "x", + "github_base_url": stub, + "installation_token": "mid-token", "installation_token_expiration": 4102444800i64 + }])) + .execute(&db) + .await?; + assert_eq!( + git_app_installations_for(&db, "deep-fork-ws").await?.0, + "fork-ws", + "the nearest holder wins over the root" + ); + Ok(()) +} diff --git a/backend/tests/schedule_fork_conflict.rs b/backend/tests/schedule_fork_conflict.rs new file mode 100644 index 0000000000..9a7ae8ff6d --- /dev/null +++ b/backend/tests/schedule_fork_conflict.rs @@ -0,0 +1,59 @@ +//! Enabling a schedule in a fork warns about every ancestor sharing the path, +//! not only the direct parent: the row was cloned down the whole chain, so the +//! cron is shared with whichever ancestors still hold a copy. + +use serde_json::json; +use sqlx::{Pool, Postgres}; + +use windmill_test_utils::*; + +async fn set_enabled( + base: &str, + path: &str, + enabled: bool, + force: bool, +) -> anyhow::Result<(u16, String)> { + let resp = reqwest::Client::new() + .post(format!("{base}/api/w/sfc-leaf/schedules/setenabled/{path}")) + .header("Authorization", "Bearer SFC_ADMIN_TOKEN") + .json(&json!({ "enabled": enabled, "force": force })) + .send() + .await?; + Ok((resp.status().as_u16(), resp.text().await?)) +} + +#[sqlx::test(fixtures("schedule_fork_conflict"))] +async fn enabling_in_a_fork_names_the_nearest_ancestor_sharing_the_path( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let base = format!("http://localhost:{}", server.addr.port()); + + let (status, body) = set_enabled(&base, "f/shared/nightly", true, false).await?; + assert_eq!(status, 400, "{body}"); + assert!( + body.contains("fork-conflict:schedule:sfc-root"), + "the middle fork deleted its copy, so the root is the one still sharing the cron: {body}" + ); + + let (status, body) = set_enabled(&base, "f/shared/own", true, false).await?; + assert_eq!( + status, 200, + "a path nothing upstream has enables freely: {body}" + ); + + sqlx::query( + "INSERT INTO schedule (workspace_id, path, edited_by, edited_at, schedule, enabled, script_path, args, is_flow, email, timezone, extra_perms, permissioned_as) + VALUES ('sfc-mid', 'f/shared/nightly', 'sfc-admin', NOW(), '0 0 0 * * *', false, 'f/shared/job', '{}', false, 'sfc-admin@windmill.dev', 'UTC', '{}', 'u/sfc-admin')", + ) + .execute(&db) + .await?; + let (status, body) = set_enabled(&base, "f/shared/nightly", true, false).await?; + assert_eq!(status, 400, "{body}"); + assert!( + body.contains("fork-conflict:schedule:sfc-mid"), + "with the parent holding a copy again, it is the nearest and gets named: {body}" + ); + Ok(()) +} diff --git a/backend/windmill-api-schedule/src/lib.rs b/backend/windmill-api-schedule/src/lib.rs index d581608c10..dfd4a6e3ba 100644 --- a/backend/windmill-api-schedule/src/lib.rs +++ b/backend/windmill-api-schedule/src/lib.rs @@ -1129,35 +1129,23 @@ pub async fn set_enabled( check_scopes(&authed, || format!("schedules:write:{}", path))?; reject_reserved_schedule_path(path)?; - // Block enabling a schedule in a fork when the parent has the same path - // (regardless of parent's enabled flag), unless force=true. Two enabled - // crons fire in lockstep; even when the parent is currently disabled the - // user is likely to re-enable it later, at which point both fire — better - // to surface that risk at every fork-side enable. There's no namespacing - // fix for schedules (Phase 3 doesn't help cron); the user has to confirm - // or point the script at fork-only side effects. + // Block enabling a schedule in a fork when an ancestor has the same path + // (regardless of its enabled flag), unless force=true. Two enabled crons + // fire in lockstep; even when the ancestor is currently disabled the user + // is likely to re-enable it later, at which point both fire — better to + // surface that risk at every fork-side enable. There's no namespacing fix + // for schedules (Phase 3 doesn't help cron); the user has to confirm or + // point the script at fork-only side effects. if payload.enabled && !payload.force { - let parent_id: Option = sqlx::query_scalar!( - "SELECT parent_workspace_id FROM workspace WHERE id = $1", - &w_id + if let Some(ancestor_id) = windmill_common::workspaces::nearest_fork_ancestor_having( + &mut *tx, "schedule", &w_id, path, ) - .fetch_optional(&mut *tx) .await? - .flatten(); - if let Some(parent_id) = parent_id { - let exists: Option = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM schedule WHERE workspace_id = $1 AND path = $2)", - &parent_id, - path, - ) - .fetch_one(&mut *tx) - .await?; - if exists == Some(true) { - return Err(Error::BadRequest(format!( - "fork-conflict:schedule:{}", - parent_id - ))); - } + { + return Err(Error::BadRequest(format!( + "fork-conflict:schedule:{}", + ancestor_id + ))); } } let before = trigger_history::snapshot_row(&mut *tx, "schedule", &w_id, path).await?; @@ -1699,9 +1687,9 @@ pub use windmill_queue::schedule::clear_schedule; #[derive(Deserialize)] pub struct SetEnabled { pub enabled: bool, - /// Bypass the parent-state warning when enabling a schedule in a fork - /// whose parent has the same path enabled. The frontend sets this after - /// the user confirms the duplicate-firing dialog. + /// Bypass the fork-conflict warning when enabling a schedule in a fork + /// while an ancestor workspace has the same path. The frontend sets this + /// after the user confirms the duplicate-firing dialog. #[serde(default)] pub force: bool, } diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index 9b6a471b2b..7cf910ef84 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -1788,6 +1788,48 @@ pub async fn workspace_with_fork_ancestors(db: &crate::DB, w_id: &str) -> Result Ok(chain) } +/// The nearest fork ancestor of `w_id` holding a row at `path` in `table`, or `None` when no +/// ancestor does (or `w_id` is not a fork). Fork creation clones trigger and schedule rows down +/// the whole chain, so a row shares its upstream identifier (Kafka group, PG slot, cron) with +/// every ancestor that still has one, not just the direct parent, which may have deleted its +/// copy since. +/// +/// Runs on the caller's connection so it sees the caller's transaction, and uncached: the +/// answer depends on the target table, not only on lineage. +/// +/// `table` is interpolated into SQL, hence `'static`: a trigger's `TABLE_NAME` or a literal, +/// never caller input. Reads lineage for any `w_id` with no authorization check, like +/// [`fork_ancestor_chain`], so the caller must already be authorized for `w_id`. +pub async fn nearest_fork_ancestor_having( + conn: &mut sqlx::PgConnection, + table: &'static str, + w_id: &str, + path: &str, +) -> Result> { + sqlx::query_scalar(&format!( + r#" + WITH RECURSIVE chain AS ( + SELECT id, parent_workspace_id, 0 AS depth + FROM workspace WHERE id = $1 + UNION ALL + SELECT w.id, w.parent_workspace_id, chain.depth + 1 + FROM workspace w + JOIN chain ON w.id = chain.parent_workspace_id + WHERE chain.depth < 20 + ) + SELECT chain.id FROM chain + JOIN {table} t ON t.workspace_id = chain.id AND t.path = $2 + WHERE chain.depth > 0 + ORDER BY chain.depth LIMIT 1 + "# + )) + .bind(w_id) + .bind(path) + .fetch_optional(&mut *conn) + .await + .map_err(|e| Error::internal_err(format!("resolving fork ancestors of {w_id}: {e:#}"))) +} + lazy_static::lazy_static! { /// workspace id -> (root workspace id, expiry ts). Read once per job start, so correctness /// rests on the invalidation rather than on the TTL: every mutation that can change the answer diff --git a/backend/windmill-trigger/src/handler.rs b/backend/windmill-trigger/src/handler.rs index 953f3a571b..66d6770354 100644 --- a/backend/windmill-trigger/src/handler.rs +++ b/backend/windmill-trigger/src/handler.rs @@ -97,14 +97,14 @@ pub trait TriggerCrud: Send + Sync + 'static { const DEPLOYMENT_NAME: &'static str; const ADDITIONAL_SELECT_FIELDS: &[&'static str] = &[]; const IS_ALLOWED_ON_CLOUD: bool; - /// Whether enabling this trigger in a fork while the parent has the same - /// path enabled is a real conflict (shared upstream resource). True for + /// Whether enabling this trigger in a fork while an ancestor workspace has + /// the same path is a real conflict (shared upstream resource). True for /// listener-based kinds where two consumers compete (Kafka group, PG slot, /// SQS queue, etc.) and for Websocket where both subscribers fire on every /// broadcast. False for kinds whose upstream identifier is implicitly /// workspace-scoped at runtime (HTTP routes, Email local_part — clones for /// the non-workspaced sub-case are filtered out, so any cloned row is - /// already collision-free vs. the parent). + /// already collision-free vs. its ancestors). const FORK_CONFLICT_ON_ENABLE: bool = true; fn get_deployed_object(path: String, parent_path: Option) -> DeployedObject; @@ -1095,54 +1095,14 @@ async fn exists_trigger( #[derive(serde::Deserialize)] struct SetTriggerModePayload { mode: TriggerMode, - /// When true, bypass the parent-state warning that would otherwise reject - /// enabling a trigger that's already enabled in the parent workspace. - /// The frontend sets this after the user confirms the duplicate-execution - /// dialog. See windmill-trigger/src/handler.rs::set_trigger_mode for the - /// full check. + /// When true, bypass the fork-conflict warning that would otherwise reject + /// enabling a trigger an ancestor workspace also has at this path. The + /// frontend sets this after the user confirms the duplicate-execution + /// dialog. See `set_trigger_mode` for the full check. #[serde(default)] force: bool, } -/// Returns the parent workspace id when this workspace is a fork *and* the -/// parent has a row at the same trigger path. Used to gate enabling a trigger -/// in a fork behind an explicit `force=true` confirmation: the fork's row was -/// cloned from the parent, so its upstream identifier (Kafka group, PG slot, -/// SQS queue URL, etc.) is shared by construction. The risk is independent of -/// the parent's current `mode`: if the parent is enabled, the two listeners -/// compete; if it's disabled, the fork can destructively take over shared -/// state (e.g. advance the PG WAL, claim an MQTT client_id) before the parent -/// re-enables. Either way, the user should be asked to confirm. -async fn parent_has_trigger( - tx: &mut PgConnection, - table_name: &str, - workspace_id: &str, - path: &str, -) -> Result> { - let parent: Option = - sqlx::query_scalar("SELECT parent_workspace_id FROM workspace WHERE id = $1") - .bind(workspace_id) - .fetch_optional(&mut *tx) - .await? - .flatten(); - let Some(parent_id) = parent else { - return Ok(None); - }; - let exists: Option = sqlx::query_scalar(&format!( - "SELECT EXISTS(SELECT 1 FROM {} WHERE workspace_id = $1 AND path = $2)", - table_name - )) - .bind(&parent_id) - .bind(path) - .fetch_one(&mut *tx) - .await?; - Ok(if exists == Some(true) { - Some(parent_id) - } else { - None - }) -} - async fn set_trigger_mode( Extension(handler): Extension>, authed: ApiAuthed, @@ -1157,22 +1117,28 @@ async fn set_trigger_mode( let mut tx = user_db.begin(&authed).await?; // Block transitioning a trigger in a fork to any mode that attaches a - // listener (Enabled or Suspended) when the parent has the same path, + // listener (Enabled or Suspended) when an ancestor has the same path, // unless the caller passes force=true. Suspended still keeps the // listener attached — it just stops auto-running queued jobs — so a // suspended fork would still split Kafka events / share a PG slot - // with the parent. The cloned upstream identifier is shared by - // construction; the risk is independent of the parent's current mode. - // Skipped for kinds where the upstream identifier is already - // workspace-scoped at runtime (HTTP, Email). + // with the ancestor. The risk is independent of the ancestor's current + // mode: enabled, the two listeners compete; disabled, the fork can + // destructively take over shared state (advance the PG WAL, claim an + // MQTT client_id) before it re-enables. Skipped for kinds where the + // upstream identifier is already workspace-scoped at runtime (HTTP, Email). if T::FORK_CONFLICT_ON_ENABLE && payload.mode != TriggerMode::Disabled && !payload.force { - if let Some(parent_id) = - parent_has_trigger(&mut *tx, T::TABLE_NAME, &workspace_id, path).await? + if let Some(ancestor_id) = windmill_common::workspaces::nearest_fork_ancestor_having( + &mut *tx, + T::TABLE_NAME, + &workspace_id, + path, + ) + .await? { return Err(Error::BadRequest(format!( "fork-conflict:{}:{}", T::TRIGGER_TYPE, - parent_id + ancestor_id ))); } } diff --git a/frontend/src/lib/components/ForkConflictModal.svelte b/frontend/src/lib/components/ForkConflictModal.svelte index e0c360320d..03afd71407 100644 --- a/frontend/src/lib/components/ForkConflictModal.svelte +++ b/frontend/src/lib/components/ForkConflictModal.svelte @@ -32,29 +32,29 @@ close(true)} onCanceled={() => close(false)} > {#if state}

    - The parent workspace ({state.parentWorkspaceId}) has the same - {state.kindLabel} configured at this path. Because this fork's row was cloned from it, the upstream - identifier is shared. + The upstream workspace ({state.upstreamWorkspaceId}) this fork + descends from has the same {state.kindLabel} configured at this path. Because this fork's row was + cloned from it, the upstream identifier is shared.

    {#if family === 'split'} If both are enabled, the two listeners will compete on the same upstream and each side will receive only a fraction of its events. {:else if family === 'duplicate'} - If both are enabled, every event will fire the script twice — once in the fork and once in - the parent. + If both are enabled, every event will fire the script twice: once in the fork and once in + the upstream workspace. {:else if family === 'slot'} The cloned replication_slot_name points at the same Postgres slot, which only allows one consumer at a time. Enabling here will either fail with "slot already active" - if the parent is enabled, or hijack the slot's WAL position if it isn't — causing the parent - to lose events when re-enabled. + if the upstream workspace is enabled, or hijack the slot's WAL position if it isn't, causing + it to lose events when re-enabled. {:else} Enabling it here may compete for the same upstream events or duplicate side effects. {/if} diff --git a/frontend/src/lib/stores.ts b/frontend/src/lib/stores.ts index 229ecff7ad..a3147b171c 100644 --- a/frontend/src/lib/stores.ts +++ b/frontend/src/lib/stores.ts @@ -242,7 +242,7 @@ export type GlobalForkModalState = { export type ForkConflictModalState = { kind: string kindLabel: string - parentWorkspaceId: string + upstreamWorkspaceId: string resolve: (proceed: boolean) => void } diff --git a/frontend/src/lib/utils/forkConflict.ts b/frontend/src/lib/utils/forkConflict.ts index d903790b44..5572e01f51 100644 --- a/frontend/src/lib/utils/forkConflict.ts +++ b/frontend/src/lib/utils/forkConflict.ts @@ -2,14 +2,15 @@ import { forkConflictModal } from '$lib/stores' /** * The backend rejects "enable" requests on triggers/schedules in a fork when - * the parent workspace has the same path enabled. The error body is shaped as - * `fork-conflict::` + * an upstream workspace (the parent, or an ancestor further up) has the same + * path. The error body is shaped as + * `fork-conflict::` * so the UI can show a tailored confirm-to-proceed dialog and re-issue the * call with `force: true` if the user agrees. */ export interface ForkConflict { kind: string - parentWorkspaceId: string + upstreamWorkspaceId: string } export function detectForkConflict(e: unknown): ForkConflict | null { @@ -20,7 +21,7 @@ export function detectForkConflict(e: unknown): ForkConflict | null { : ((body as any)?.error?.message ?? (body as any)?.message ?? (e as any)?.message ?? '') const m = String(raw).match(/fork-conflict:([^:]+):(.+)/) if (!m) return null - return { kind: m[1], parentWorkspaceId: m[2].trim() } + return { kind: m[1], upstreamWorkspaceId: m[2].trim() } } /** @@ -30,11 +31,11 @@ export function detectForkConflict(e: unknown): ForkConflict | null { * on two rows in quick succession), resolve the older promise to false so * the prior caller doesn't hang. */ -function askForkConflictConfirm(kind: string, kindLabel: string, parentWorkspaceId: string) { +function askForkConflictConfirm(kind: string, kindLabel: string, upstreamWorkspaceId: string) { return new Promise((resolve) => { const previous = forkConflictModal.val previous?.resolve(false) - forkConflictModal.val = { kind, kindLabel, parentWorkspaceId, resolve } + forkConflictModal.val = { kind, kindLabel, upstreamWorkspaceId, resolve } }) } @@ -64,7 +65,7 @@ export async function withForkConflictRetry( const proceed = await askForkConflictConfirm( conflict.kind, kindLabel, - conflict.parentWorkspaceId + conflict.upstreamWorkspaceId ) // User explicitly dismissed the modal — treat as a silent no-op so the // caller's catch block doesn't pop a redundant error toast. From b51c0eabbe774c78a9cbf824a3df6528d970b8f6 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Wed, 16 Sep 2026 10:46:15 +0200 Subject: [PATCH 143/144] feat: stream reasoning summaries in AI agent Responses API steps (#11124) * feat: stream reasoning summaries in AI agent Responses API steps Co-Authored-By: Claude Opus 5 * fix: retry without a reasoning summary a strict gateway rejects Co-Authored-By: Claude Opus 5 * fix: request the reasoning summary whether or not the step streams Co-Authored-By: Claude Opus 5 * feat: return the OpenAI reasoning summary in the agent step's reasoning Co-Authored-By: Claude Opus 5 * fix: retry without a reasoning summary a gateway rejects with 422 Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Opus 5 --- .../windmill-ai/src/providers/anthropic.rs | 1 + backend/windmill-ai/src/providers/openai.rs | 194 ++++++++++++++++-- backend/windmill-ai/src/query_builder.rs | 3 + backend/windmill-ai/src/sse.rs | 63 ++++++ backend/windmill-ai/src/types.rs | 3 +- backend/windmill-worker/src/ai_executor.rs | 23 ++- 6 files changed, 269 insertions(+), 18 deletions(-) diff --git a/backend/windmill-ai/src/providers/anthropic.rs b/backend/windmill-ai/src/providers/anthropic.rs index 038aef2c2e..0bb7316dcb 100644 --- a/backend/windmill-ai/src/providers/anthropic.rs +++ b/backend/windmill-ai/src/providers/anthropic.rs @@ -902,6 +902,7 @@ mod tests { attachments: None, has_websearch: false, prompt_cache_key: None, + reasoning_summary: false, }; AnthropicQueryBuilder::new(AIProvider::Anthropic, platform) diff --git a/backend/windmill-ai/src/providers/openai.rs b/backend/windmill-ai/src/providers/openai.rs index 8fa65dca21..14c9fdb4d3 100644 --- a/backend/windmill-ai/src/providers/openai.rs +++ b/backend/windmill-ai/src/providers/openai.rs @@ -1,6 +1,7 @@ use crate::{ ai_providers::AIProvider, ai_types::OpenAIToolCall, + credentials::ProviderCredentials, image_handler::{prepare_messages_for_api, s3_object_to_content_part}, proxy::{build_openai_compatible_proxy_request, ProxyBuildArgs, ProxyRequest}, query_builder::{BuildRequestArgs, ParsedResponse, QueryBuilder, StreamEventSink}, @@ -11,7 +12,14 @@ use crate::{ use async_trait::async_trait; use serde::{Deserialize, Serialize}; use serde_json::value::RawValue; -use windmill_common::{client::AuthedClient, error::Error}; +use std::{ + collections::{BTreeMap, HashMap}, + hash::{DefaultHasher, Hash, Hasher}, + time::{Duration, Instant}, +}; +use windmill_common::{cache::Cache, client::AuthedClient, error::Error}; + +use super::REASONING_OFF_SENTINEL; // Responses API structures #[derive(Deserialize)] @@ -192,13 +200,79 @@ pub struct ResponsesApiTextFormat { pub format: ResponsesApiTextFormatConfig, } -/// Reasoning config for the Responses API (`reasoning: { effort }`). -/// The summary is intentionally not requested, mirroring the copilot chat: OpenAI -/// gates reasoning summaries behind organization verification, so asking for one -/// would fail the request for unverified orgs. +/// Reasoning config for the Responses API (`reasoning: { effort, summary }`). #[derive(Serialize)] pub struct ResponsesApiReasoning { pub effort: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub summary: Option, +} + +lazy_static::lazy_static! { + /// Refused reasoning summaries, so later requests skip asking instead of paying a + /// rejected call each. A refusal belongs to the organization the request bills or to the + /// model, so the key holds the model and everything that authenticates (the API key and + /// the resource headers, which can carry it instead). Entries expire as an org gets verified. + static ref REASONING_SUMMARY_UNAVAILABLE: Cache<(String, u64), Instant> = Cache::new(500); +} + +const REASONING_SUMMARY_UNAVAILABLE_TTL: Duration = Duration::from_secs(3600); + +fn reasoning_summary_cache_key( + base_url: &str, + model: &str, + api_key: Option<&str>, + custom_headers: &HashMap, +) -> (String, u64) { + let mut hasher = DefaultHasher::new(); + model.hash(&mut hasher); + api_key.hash(&mut hasher); + // Sorted: two maps with the same entries can iterate them in different orders. + custom_headers + .iter() + .collect::>() + .hash(&mut hasher); + (base_url.to_string(), hasher.finish()) +} + +fn credentials_cache_key(credentials: &ProviderCredentials, model: &str) -> (String, u64) { + reasoning_summary_cache_key( + &credentials.base_url, + model, + credentials.api_key.as_deref(), + &credentials.custom_headers, + ) +} + +/// Whether this model is known to be refused reasoning summaries with these credentials. +pub fn is_reasoning_summary_unavailable(credentials: &ProviderCredentials, model: &str) -> bool { + REASONING_SUMMARY_UNAVAILABLE + .get(&credentials_cache_key(credentials, model)) + .is_some_and(|learned_at| learned_at.elapsed() < REASONING_SUMMARY_UNAVAILABLE_TTL) +} + +/// Record that this model was refused a reasoning summary with these credentials. +pub fn remember_reasoning_summary_unavailable(credentials: &ProviderCredentials, model: &str) { + REASONING_SUMMARY_UNAVAILABLE.insert(credentials_cache_key(credentials, model), Instant::now()); +} + +/// Whether a rejected request was refused over its reasoning summary, e.g. `Your +/// organization must be verified to generate reasoning summaries` (param +/// `reasoning.summary`). An OpenAI-kind resource can also point at a gateway that validates +/// the body strictly and names only the unknown `summary` property. +pub fn rejects_reasoning_summary(status: u16, body: &str) -> bool { + // 422 is how FastAPI-based gateways reject a body that fails validation. + if !matches!(status, 400 | 403 | 422) { + return false; + } + let body = body.to_lowercase(); + let unknown_field = body.contains("additional properties are not allowed") + || body.contains("unrecognized request argument") + || body.contains("extra inputs are not permitted"); + body.contains("reasoning.summary") + || body.contains("verified to generate reasoning summar") + || body.contains("verified to stream reasoning summar") + || (unknown_field && body.contains("summary")) } #[derive(Serialize)] @@ -435,9 +509,13 @@ impl OpenAIQueryBuilder { tools, stream: Some(true), temperature: args.temperature, - reasoning: args - .reasoning_effort - .map(|effort| ResponsesApiReasoning { effort: effort.to_string() }), + reasoning: args.reasoning_effort.map(|effort| ResponsesApiReasoning { + effort: effort.to_string(), + // A request that does not reason has nothing to summarize, yet asking still + // gets an unverified organization's request rejected. + summary: (args.reasoning_summary && effort != REASONING_OFF_SENTINEL) + .then(|| "auto".to_string()), + }), max_output_tokens: args.max_tokens, text, prompt_cache_key: args.prompt_cache_key, @@ -538,9 +616,8 @@ impl QueryBuilder for OpenAIQueryBuilder { } else { Some(parser.accumulated_content) }, - // The Responses stream has no reasoning-summary event in - // `OpenAIResponsesSSEEvent`, so nothing thinks out loud on this path yet. - reasoning: None, + reasoning: (!parser.accumulated_reasoning.is_empty()) + .then_some(parser.accumulated_reasoning), tool_calls: parser.accumulated_tool_calls.into_values().collect(), events_str: Some(parser.events_str), annotations: parser.annotations, @@ -640,8 +717,11 @@ mod tests { } } - async fn build_text_body(messages: &[OpenAIMessage], system_prompt: Option<&str>) -> String { - let args = BuildRequestArgs { + fn text_args<'a>( + messages: &'a [OpenAIMessage], + system_prompt: Option<&'a str>, + ) -> BuildRequestArgs<'a> { + BuildRequestArgs { messages, tools: None, model: "gpt-5", @@ -655,14 +735,98 @@ mod tests { attachments: None, has_websearch: false, prompt_cache_key: Some(PROMPT_CACHE_KEY), - }; + reasoning_summary: true, + } + } + async fn build_body(args: &BuildRequestArgs<'_>) -> String { OpenAIQueryBuilder::new(AIProvider::OpenAI) - .build_request(&args, &client(), "test-workspace") + .build_request(args, &client(), "test-workspace") .await .unwrap() } + async fn build_text_body(messages: &[OpenAIMessage], system_prompt: Option<&str>) -> String { + build_body(&text_args(messages, system_prompt)).await + } + + async fn reasoning_of(effort: Option<&str>, reasoning_summary: bool) -> serde_json::Value { + let messages = vec![message("user", "hi")]; + let args = BuildRequestArgs { + reasoning_effort: effort, + reasoning_summary, + ..text_args(&messages, None) + }; + let request: serde_json::Value = serde_json::from_str(&build_body(&args).await).unwrap(); + request["reasoning"].clone() + } + + #[tokio::test] + async fn requests_a_reasoning_summary_only_when_the_model_reasons() { + assert_eq!( + reasoning_of(Some("high"), true).await, + serde_json::json!({ "effort": "high", "summary": "auto" }) + ); + assert_eq!( + reasoning_of(Some("none"), true).await, + serde_json::json!({ "effort": "none" }) + ); + assert_eq!( + reasoning_of(Some("high"), false).await, + serde_json::json!({ "effort": "high" }) + ); + assert!(reasoning_of(None, true).await.is_null()); + } + + #[test] + fn recognizes_a_refused_reasoning_summary() { + let unverified = r#"{"error":{"message":"Your organization must be verified to generate reasoning summaries. Please go to: https://platform.openai.com/settings/organization/general and click on Verify Organization.","type":"invalid_request_error","param":"reasoning.summary","code":"unsupported_value"}}"#; + assert!(rejects_reasoning_summary(400, unverified)); + assert!(!rejects_reasoning_summary(500, unverified)); + assert!(rejects_reasoning_summary( + 400, + r#"{"detail":"Additional properties are not allowed ('summary' was unexpected)"}"# + )); + assert!(rejects_reasoning_summary( + 422, + r#"{"detail":[{"type":"extra_forbidden","loc":["body","reasoning","summary"],"msg":"Extra inputs are not permitted","input":"auto"}]}"# + )); + assert!(!rejects_reasoning_summary( + 400, + r#"{"error":{"message":"Invalid 'prompt_cache_key': string too long","param":"prompt_cache_key"}}"# + )); + } + + /// A resource can authenticate through `headers` with no API key: one organization's + /// refusal must not withhold summaries from another's. + #[test] + fn keys_a_refused_summary_by_the_header_credential() { + let headers = |pairs: &[(&str, &str)]| { + pairs + .iter() + .map(|(name, value)| (name.to_string(), value.to_string())) + .collect::>() + }; + let url = "https://api.openai.com/v1"; + let org_a = headers(&[("Authorization", "Bearer org-a"), ("X-Trace", "1")]); + let org_a_reordered = headers(&[("X-Trace", "1"), ("Authorization", "Bearer org-a")]); + let org_b = headers(&[("Authorization", "Bearer org-b"), ("X-Trace", "1")]); + + assert_ne!( + reasoning_summary_cache_key(url, "gpt-5", None, &org_a), + reasoning_summary_cache_key(url, "gpt-5", None, &org_b) + ); + assert_eq!( + reasoning_summary_cache_key(url, "gpt-5", None, &org_a), + reasoning_summary_cache_key(url, "gpt-5", None, &org_a_reordered) + ); + // A model can refuse summaries that another model on the same credentials streams. + assert_ne!( + reasoning_summary_cache_key(url, "gpt-5", None, &org_a), + reasoning_summary_cache_key(url, "gpt-5-mini", None, &org_a) + ); + } + /// The worker prepends the system prompt as a system message *and* passes it as /// `system_prompt`; the request must still carry it exactly once. #[tokio::test] diff --git a/backend/windmill-ai/src/query_builder.rs b/backend/windmill-ai/src/query_builder.rs index 60ece300a3..3281b51cd0 100644 --- a/backend/windmill-ai/src/query_builder.rs +++ b/backend/windmill-ai/src/query_builder.rs @@ -27,6 +27,9 @@ pub struct BuildRequestArgs<'a> { /// the prefix (the step), never from the request. `None` retries a key the /// endpoint rejected. pub prompt_cache_key: Option<&'a str>, + /// Ask for a summary of the model's reasoning where the provider streams one. + /// `false` once the provider refused summaries to these credentials. + pub reasoning_summary: bool, } /// Response from AI provider diff --git a/backend/windmill-ai/src/sse.rs b/backend/windmill-ai/src/sse.rs index 986642fd7e..054baeb5de 100644 --- a/backend/windmill-ai/src/sse.rs +++ b/backend/windmill-ai/src/sse.rs @@ -815,6 +815,14 @@ pub enum OpenAIResponsesSSEEvent { #[serde(rename = "response.output_text.annotation.added")] AnnotationAdded { annotation: OpenAIUrlCitationEvent }, + /// A new reasoning summary part starts (only sent when `reasoning.summary` was requested) + #[serde(rename = "response.reasoning_summary_part.added")] + ReasoningSummaryPartAdded {}, + + /// Reasoning summary text delta + #[serde(rename = "response.reasoning_summary_text.delta")] + ReasoningSummaryTextDelta { delta: String }, + /// Catch-all for unknown event types #[serde(other)] Other, @@ -823,6 +831,8 @@ pub enum OpenAIResponsesSSEEvent { /// OpenAI Responses API SSE Parser for streaming responses pub struct OpenAIResponsesSSEParser { pub accumulated_content: String, + /// The reasoning summary streamed before the answer, kept so it can be stored with it. + pub accumulated_reasoning: String, pub accumulated_tool_calls: HashMap, /// Maps item_id -> (name, call_id) for function calls tool_call_metadata: HashMap, @@ -836,12 +846,15 @@ pub struct OpenAIResponsesSSEParser { pub used_websearch: bool, /// Token usage from response.completed event pub usage: Option, + /// Reasoning summary parts seen so far, to separate them as paragraphs + reasoning_summary_parts: usize, } impl OpenAIResponsesSSEParser { pub fn new(stream_event_processor: Box) -> Self { Self { accumulated_content: String::new(), + accumulated_reasoning: String::new(), accumulated_tool_calls: HashMap::new(), tool_call_metadata: HashMap::new(), tool_call_arguments: HashMap::new(), @@ -850,6 +863,7 @@ impl OpenAIResponsesSSEParser { annotations: Vec::new(), used_websearch: false, usage: None, + reasoning_summary_parts: 0, } } } @@ -959,6 +973,28 @@ impl SSEParser for OpenAIResponsesSSEParser { } } + OpenAIResponsesSSEEvent::ReasoningSummaryPartAdded {} => { + self.reasoning_summary_parts += 1; + if self.reasoning_summary_parts > 1 { + self.accumulated_reasoning.push_str("\n\n"); + let event = + StreamingEvent::ReasoningTokenDelta { content: "\n\n".to_string() }; + self.stream_event_processor + .send(event, &mut self.events_str) + .await?; + } + } + + OpenAIResponsesSSEEvent::ReasoningSummaryTextDelta { delta } => { + if !delta.is_empty() { + self.accumulated_reasoning.push_str(&delta); + let event = StreamingEvent::ReasoningTokenDelta { content: delta }; + self.stream_event_processor + .send(event, &mut self.events_str) + .await?; + } + } + // Ignore other event types OpenAIResponsesSSEEvent::Done {} | OpenAIResponsesSSEEvent::Created {} @@ -1015,6 +1051,33 @@ mod tests { assert_eq!(token_usage.total_tokens, Some(4821)); } + struct ReasoningSink; + + #[async_trait::async_trait] + impl StreamEventSink for ReasoningSink { + async fn send(&self, event: StreamingEvent, events_str: &mut String) -> Result<(), Error> { + if let StreamingEvent::ReasoningTokenDelta { content } = event { + events_str.push_str(&content); + } + Ok(()) + } + } + + #[tokio::test] + async fn streams_openai_responses_reasoning_summary_parts_as_paragraphs() { + let mut parser = OpenAIResponsesSSEParser::new(Box::new(ReasoningSink)); + for data in [ + r#"{"type":"response.reasoning_summary_part.added","item_id":"rs_1","output_index":0,"summary_index":0,"part":{"type":"summary_text","text":""}}"#, + r#"{"type":"response.reasoning_summary_text.delta","item_id":"rs_1","output_index":0,"summary_index":0,"delta":"**Planning**"}"#, + r#"{"type":"response.reasoning_summary_part.added","item_id":"rs_1","output_index":0,"summary_index":1,"part":{"type":"summary_text","text":""}}"#, + r#"{"type":"response.reasoning_summary_text.delta","item_id":"rs_1","output_index":0,"summary_index":1,"delta":"Then answer"}"#, + ] { + parser.parse_event_data(data).await.unwrap(); + } + assert_eq!(parser.events_str, "**Planning**\n\nThen answer"); + assert_eq!(parser.accumulated_reasoning, "**Planning**\n\nThen answer"); + } + #[test] fn openai_delta_parses_reasoning_content() { // DeepSeek and similar stream reasoning under `reasoning_content`. diff --git a/backend/windmill-ai/src/types.rs b/backend/windmill-ai/src/types.rs index 8dfb9af1c6..ae85718e8a 100644 --- a/backend/windmill-ai/src/types.rs +++ b/backend/windmill-ai/src/types.rs @@ -377,8 +377,7 @@ pub struct AIAgentResult<'a> { /// The model's thinking across every iteration of the loop, in order, blank-line /// separated. Present whenever the provider's parser surfaced any, whether or not /// the step streams, so a downstream step never has to pick it out of `wm_stream`. - /// Absent when the model thought nothing, and on the OpenAI Responses path, whose - /// parser does not return reasoning yet. + /// Absent when the model thought nothing. #[serde(skip_serializing_if = "Option::is_none")] pub reasoning: Option, #[serde(skip_serializing_if = "Option::is_none")] diff --git a/backend/windmill-worker/src/ai_executor.rs b/backend/windmill-worker/src/ai_executor.rs index debfbeff39..156369e095 100644 --- a/backend/windmill-worker/src/ai_executor.rs +++ b/backend/windmill-worker/src/ai_executor.rs @@ -26,6 +26,10 @@ use windmill_ai::{ image_handler::upload_image_to_s3, providers::{ create_chat_completions_query_builder, create_query_builder, is_chat_completions_only, + openai::{ + is_reasoning_summary_unavailable, rejects_reasoning_summary, + remember_reasoning_summary_unavailable, + }, remember_chat_completions_only, }, proxy::{ @@ -1300,6 +1304,10 @@ pub async fn run_agent( attachments: args.user_attachments.as_deref(), has_websearch, prompt_cache_key: include_prompt_cache_key.then_some(prompt_cache_key.as_str()), + reasoning_summary: !is_reasoning_summary_unavailable( + &credentials, + args.provider.get_model(), + ), }; // A worker cannot run the client credentials exchange, so an OAuth resource @@ -1350,7 +1358,8 @@ pub async fn run_agent( // An endpoint can reject the request shape rather than the model: // `stream_options` and `prompt_cache_key`, which not every OpenAI-compatible - // gateway accepts, and the route itself, when an Azure resource is outside + // gateway accepts, a reasoning summary, which OpenAI refuses to unverified + // organizations, and the route itself, when an Azure resource is outside // the Responses API's model/region matrix. Each is retried once with that // part dropped. // Set where the route is found to be absent, and read once the fallback has @@ -1407,6 +1416,9 @@ pub async fn run_agent( && status.as_u16() == 400 && text.contains("prompt_cache_key"); + let summary_refused = build_args.reasoning_summary + && rejects_reasoning_summary(status.as_u16(), &text); + // Only the first call of the step may re-route: an endpoint that // does not serve this API rejects that one already, whereas a // rejection once the conversation is under way is about the @@ -1430,6 +1442,15 @@ pub async fn run_agent( ); include_prompt_cache_key = false; build_args.prompt_cache_key = None; + } else if summary_refused { + tracing::info!( + "Retrying request without the reasoning summary the endpoint refused" + ); + remember_reasoning_summary_unavailable( + &credentials, + args.provider.get_model(), + ); + build_args.reasoning_summary = false; } else if route_unserved { tracing::info!( "Endpoint rejected the request ({}), falling back to chat/completions", From 54553b2add6941395f03ca34ee24cf335a3b23c2 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Wed, 16 Sep 2026 10:53:24 +0200 Subject: [PATCH 144/144] fix(cli): resolve lockgen imports through modules a push leaves alone (#11160) Co-authored-by: Claude Opus 5 --- cli/src/commands/sync/sync.ts | 42 ++++++++++ cli/src/utils/dependency_tree.ts | 81 ++++++++++++++++++- cli/test/dependency_tree_unit.test.ts | 64 ++++++++++++++- .../sync_push_auto_metadata_repro.test.ts | 75 +++++++++++++++++ 4 files changed, 259 insertions(+), 3 deletions(-) diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index 38b9e87f84..138a346594 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -128,6 +128,8 @@ import { } from "../../utils/metadata.ts"; import { DoubleLinkedDependencyTree, + LocalScripts, + resolvePlaceholdersFromLocal, uploadScripts, } from "../../utils/dependency_tree.ts"; import { @@ -176,6 +178,7 @@ import { isDbtModulePath, isDbtGeneratedPath, isModuleEntryPoint, + scriptPathToRemotePath, getScriptBasePathFromModulePath, hasWrongFormatSuffix, DBT_DESCRIPTOR_NAME, @@ -3385,6 +3388,37 @@ async function addToChangedIfNotExists(p: string, tracker: ChangeTracker) { } } +/** + * Index the checkout's standalone scripts by the remote path a relative import + * resolves to, reusing the content the local/remote diff already read. + * + * Same classification as `addToChangedIfNotExists`: a flow or app inline script + * is not addressable as an import target, and a module bundle is addressed by + * its entry point. + */ +function localScriptsByRemotePath( + localMap: Record, +): LocalScripts { + const byRemotePath: LocalScripts = new Map(); + for (const [p, content] of Object.entries(localMap)) { + if (isScriptModulePath(p)) { + if (!isModuleEntryPoint(p)) continue; + } else if ( + !hasScriptExt(p) || + isDatatableMigrationPath(p) || + isFileResource(p) || + isFilesetResource(p) || + isFlowPath(p) || + isAppPath(p) || + isRawAppPath(p) + ) { + continue; + } + byRemotePath.set(scriptPathToRemotePath(p), { localPath: p, content }); + } + return byRemotePath; +} + export async function buildTracker(changes: Change[]) { const tracker: ChangeTracker = { scripts: [], @@ -5074,6 +5108,14 @@ export async function push( } if (autoRegenerate && tree) { + // Pass 1 only ever walks the change set, so anything imported through a + // module the push leaves alone is still a dead end here. + await resolvePlaceholdersFromLocal( + tree, + localScriptsByRemotePath(localMap), + opts.defaultTs, + ); + // Propagate staleness through imports + upload script content to // raw_script_temp so the dep job can resolve cross-folder relative imports // via temp_script_refs (instead of hitting 404s for not-yet-deployed diff --git a/cli/src/utils/dependency_tree.ts b/cli/src/utils/dependency_tree.ts index b0b75e659e..959a77b478 100644 --- a/cli/src/utils/dependency_tree.ts +++ b/cli/src/utils/dependency_tree.ts @@ -5,7 +5,7 @@ import { Workspace } from "../commands/workspace/workspace.ts"; import * as wmill from "../../gen/services.gen.ts"; import type { ScriptLang } from "../../gen/types.gen.ts"; -import { ScriptLanguage } from "./script_common.ts"; +import { ScriptLanguage, inferContentTypeFromFilePath } from "./script_common.ts"; import { filterWorkspaceDependencies, generateScriptHash, @@ -14,6 +14,65 @@ import { updateMetadataGlobalLock, } from "./metadata.ts"; import { generateHash } from "./utils.ts"; +import { extractRelativeImports } from "./relative_imports.ts"; + +/** A local script file, keyed in `LocalScripts` by its Windmill remote path. */ +export interface LocalScriptSource { + localPath: string; + content: string; +} +export type LocalScripts = Map; + +/** + * Give every import target that is only a placeholder its local content and its + * own imports, so the graph continues through it. A tree seeded from a subset of + * the checkout otherwise dead-ends at any module outside that subset — a + * re-export barrel needing no edit, typically — hiding what it re-exports from + * `getTempScriptRefs`, which then resolves it against the deployed copy. + */ +export async function resolvePlaceholdersFromLocal( + tree: DoubleLinkedDependencyTree, + localScripts: LocalScripts, + defaultTs: "bun" | "deno" | undefined +): Promise { + // A module resolved here can expose placeholders of its own (a barrel behind + // a barrel), so keep going until a round resolves nothing. + for (;;) { + let resolved = false; + for (const remotePath of tree.placeholderPaths()) { + const local = localScripts.get(remotePath); + if (!local) continue; + let language: ScriptLanguage; + try { + language = inferContentTypeFromFilePath(local.localPath, defaultTs); + } catch { + // A bare `.sql` names no dialect, so its imports cannot be read here. + continue; + } + const imports = await extractRelativeImports( + local.content, + remotePath, + language + ); + // Never directly stale: it is outside the change set, so nothing relocks + // it. It is here to carry edges, and to be uploaded if it differs from + // what is deployed. + await tree.addNode( + remotePath, + local.content, + language, + "", + imports, + "script", + remotePath, + local.localPath, + false + ); + resolved = true; + } + if (!resolved) break; + } +} /** * Diff local scripts against deployed versions, upload only those that differ. @@ -97,6 +156,9 @@ interface DependencyNode { originalPath: string; // Original path passed to handler (with extension for scripts) isRawApp?: boolean; // Only set for apps isDirectlyStale: boolean; // True if this item's content changed (vs transitively stale) + // True while the node exists only because something imports it, so it carries + // no content and no imports of its own. + isPlaceholder: boolean; } export class DoubleLinkedDependencyTree { @@ -130,9 +192,11 @@ export class DoubleLinkedDependencyTree { content: "", stalenessHash: "", language: "deno", metadata: "", imports: new Set(), importedBy: new Set(), itemType: "script", folder: "", originalPath: "", isDirectlyStale: false, + isPlaceholder: true, }); } const node = this.nodes.get(path)!; + node.isPlaceholder = false; node.content = content; node.stalenessHash = stalenessHash; node.language = language; @@ -155,7 +219,7 @@ export class DoubleLinkedDependencyTree { stalenessHash: "", language: depsInfo?.language ?? "deno", metadata: "", imports: new Set(), importedBy: new Set(), itemType: "dependencies", folder: "", originalPath: depsPath, - isDirectlyStale: !isUpToDate, + isDirectlyStale: !isUpToDate, isPlaceholder: false, }); } } @@ -169,6 +233,7 @@ export class DoubleLinkedDependencyTree { content: "", stalenessHash: "", language: "deno", metadata: "", imports: new Set(), importedBy: new Set(), itemType: "script", folder: "", originalPath: "", isDirectlyStale: false, + isPlaceholder: true, }); } this.nodes.get(importPath)!.importedBy.add(path); @@ -309,6 +374,18 @@ export class DoubleLinkedDependencyTree { return this.nodes.keys(); } + /** + * Paths that exist only as somebody's import target, so the traversal stops + * at them instead of continuing into what they themselves import. + */ + placeholderPaths(): string[] { + const result: string[] = []; + for (const [path, node] of this.nodes.entries()) { + if (node.isPlaceholder) result.push(path); + } + return result; + } + /** * Returns paths of all stale nodes (those with a staleReason). */ diff --git a/cli/test/dependency_tree_unit.test.ts b/cli/test/dependency_tree_unit.test.ts index 1a92bb9894..8e9c733c00 100644 --- a/cli/test/dependency_tree_unit.test.ts +++ b/cli/test/dependency_tree_unit.test.ts @@ -2,7 +2,10 @@ import { expect, test } from "bun:test"; import { mkdtemp, rm } from "node:fs/promises"; import os from "node:os"; import * as path from "node:path"; -import { DoubleLinkedDependencyTree } from "../src/utils/dependency_tree.ts"; +import { + DoubleLinkedDependencyTree, + resolvePlaceholdersFromLocal, +} from "../src/utils/dependency_tree.ts"; // addNode consults wmill-lock.yaml from cwd for workspace deps; run inside a // temp dir so the test never reads/writes the repo's own lock file. @@ -104,3 +107,62 @@ test("getAllTempScriptRefs is a superset of getTempScriptRefs for any node", asy }); }); }); + +// Two barrels deep so the fixpoint matters: resolving the first one is what +// puts the second in the tree, and only a further round reaches the leaf. +test("resolvePlaceholdersFromLocal walks the graph through unresolved barrels", async () => { + await withTempDir(async () => { + const tree = new DoubleLinkedDependencyTree(); + await tree.addNode( + "f/app/consumer", + `import { subtract } from "../barrel/index.ts"`, + "bun", + "", + ["f/barrel/index"], + "script", + "f/app/consumer", + "f/app/consumer.ts", + true, + ); + await tree.addNode( + "f/barrel/helper", + "export function subtract(a: number, b: number) { return a - b }", + "bun", + "", + [], + "script", + "f/barrel/helper", + "f/barrel/helper.ts", + true, + ); + // Neither barrel is in the change set, so both are bare import targets. + expect(tree.getTempScriptRefs("f/app/consumer")).toEqual({}); + + await resolvePlaceholdersFromLocal( + tree, + new Map([ + [ + "f/barrel/index", + { + localPath: "f/barrel/index.ts", + content: `export * from "./mid.ts"`, + }, + ], + [ + "f/barrel/mid", + { + localPath: "f/barrel/mid.ts", + content: `export * from "./helper.ts"`, + }, + ], + ]), + "bun", + ); + + // Only the leaf diverged from deployed, so only it was uploaded. + tree.setContentHash("f/barrel/helper", "hash_helper"); + expect(tree.getTempScriptRefs("f/app/consumer")).toEqual({ + "f/barrel/helper": "hash_helper", + }); + }); +}); diff --git a/cli/test/sync_push_auto_metadata_repro.test.ts b/cli/test/sync_push_auto_metadata_repro.test.ts index 4854198d64..38da094a37 100644 --- a/cli/test/sync_push_auto_metadata_repro.test.ts +++ b/cli/test/sync_push_auto_metadata_repro.test.ts @@ -268,3 +268,78 @@ test( }); }, ); + +// The importer and the leaf change; the barrel between them does not, so it is +// absent from the push's change set. See `resolvePlaceholdersFromLocal`. +test( + "sync push --auto-metadata succeeds when a changed leaf sits behind an unchanged barrel", + { timeout: 180000 }, + async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile(`${tempDir}/wmill.yaml`, wmillYaml); + + await createLocalScript( + tempDir, + "f/barrel", + "helper", + "bun", + `export function add(a: number, b: number) { return a + b; }\n`, + ); + await createLocalScript( + tempDir, + "f/barrel", + "index", + "bun", + `export * from "./helper.ts";\n`, + ); + await createLocalScript( + tempDir, + "f/app", + "consumer", + "bun", + `import { add } from "../barrel/index.ts"; +export async function main() { return add(1, 2); } +`, + ); + + const deploy = await backend.runCLICommand( + ["sync", "push", "--yes", "--auto-metadata"], + tempDir, + ); + if (deploy.code !== 0) { + console.log("STDOUT:", deploy.stdout); + console.log("STDERR:", deploy.stderr); + } + expect(deploy.code).toBe(0); + + // Add an export to the leaf and use it from the importer. The barrel + // re-exports it already, so it stays byte-identical and out of the push. + await writeFile( + `${tempDir}/f/barrel/helper.ts`, + `export function add(a: number, b: number) { return a + b; } +export function subtract(a: number, b: number) { return a - b; } +`, + ); + await writeFile( + `${tempDir}/f/app/consumer.ts`, + `import { subtract } from "../barrel/index.ts"; +export async function main() { return subtract(3, 1); } +`, + ); + + const result = await backend.runCLICommand( + ["sync", "push", "--yes", "--auto-metadata"], + tempDir, + ); + if (result.code !== 0) { + console.log("STDOUT:", result.stdout); + console.log("STDERR:", result.stderr); + } + expect(result.code).toBe(0); + + const combined = result.stdout + result.stderr; + expect(combined).not.toContain("No matching export"); + expect(combined).not.toContain("Failed to generate lockfile"); + }); + }, +);

    Triggers to deploy
    +
    +
    + + {#if trigger.isPrimary} + + {/if} +
    + +
    + +
    +
    +     		+ {#if permission === 'deploy'} +
    + toggleTrigger(trigger, e.detail)} + > + {#snippet children({ item })} + + + {/snippet} + +
    + {:else if permission === 'admin-only'} + Admin only + {:else if permission === 'invalid-config'} + Invalid config + {/if} +
    Agents to deploy
    +
    + +
    +
    + + + {agent.path} + + {#if agent.noDeployed} + Never deployed + {/if} +
    + {#if agent.noDeployed && !isSelectedAgent} + + + Never deployed, so the flow will not run until this agent is deployed. + + {/if} +
    +
    +     		+ {#if permission.state === 'deploy'} +
    + toggleAgent(agent, e.detail)} + > + {#snippet children({ item })} + + + {/snippet} + +
    + {:else if permission.state === 'read-only'} + + Read-only + + {:else} + Invalid config + {/if} +
    Triggers to deploy
    -
    -
    - - {#if trigger.isPrimary} - - {/if} -
    -
    - -
    -
    -     		- {#if permission === 'deploy'} -
    - toggleTrigger(trigger, e.detail)} - > - {#snippet children({ item })} - - - {/snippet} - -
    - {:else if permission === 'admin-only'} - Admin only - {:else if permission === 'invalid-config'} - Invalid config - {/if} -
    - No draft triggers found -
    + +