From ee9e550a484fda286eeab43b7db5f314b8b2d0d9 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 7 Sep 2026 13:57:09 +0000 Subject: [PATCH 01/15] feat(git-sync): sync extra_perms for variables (#11004) * feat(git-sync): sync extra_perms for variables Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Hvv5B8VP5Di4dbcCiVyZyE * refactor: trim the variable ACL-sync comment to the 4-line limit Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Hvv5B8VP5Di4dbcCiVyZyE * test: cover the revoke direction of variable extra_perms sync Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Hvv5B8VP5Di4dbcCiVyZyE --------- Co-authored-by: Claude Opus 5 (1M context) --- .../windmill-api-groups/src/granular_acls.rs | 26 ++++ backend/windmill-api/src/workspaces_export.rs | 11 +- cli/src/commands/sync/pull.ts | 2 +- cli/src/commands/variable/variable.ts | 72 +++++++---- cli/test/variable_resource_push.test.ts | 120 ++++++++++++++++++ 5 files changed, 197 insertions(+), 34 deletions(-) diff --git a/backend/windmill-api-groups/src/granular_acls.rs b/backend/windmill-api-groups/src/granular_acls.rs index c6f88dace5..07b8004d83 100644 --- a/backend/windmill-api-groups/src/granular_acls.rs +++ b/backend/windmill-api-groups/src/granular_acls.rs @@ -318,6 +318,19 @@ async fn add_granular_acl( ) .await? } + "variable" => { + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + DeployedObject::Variable { path: path.to_string(), parent_path: None }, + Some(format!("Variable '{}' changed permissions", path)), + true, + None, + ) + .await? + } _ => (), } @@ -528,6 +541,19 @@ async fn remove_granular_acl( ) .await? } + "variable" => { + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + DeployedObject::Variable { path: path.to_string(), parent_path: None }, + Some(format!("Variable '{}' changed permissions", path)), + true, + None, + ) + .await? + } _ => (), } } diff --git a/backend/windmill-api/src/workspaces_export.rs b/backend/windmill-api/src/workspaces_export.rs index 05cd354cd4..1c22981ee0 100644 --- a/backend/windmill-api/src/workspaces_export.rs +++ b/backend/windmill-api/src/workspaces_export.rs @@ -346,7 +346,7 @@ pub(crate) struct ArchiveQueryParams { default_ts: Option, /// Settings format version: "v1" (default) returns legacy flat format, "v2" returns grouped format settings_version: Option, - /// Opt-in: include `extra_perms` on flow / script / app rows. Default `false` + /// Opt-in: include `extra_perms` on script / flow / app / variable rows. Default `false` /// so cross-workspace tarball imports do not carry over ACLs referring to /// identities that may not exist in the target workspace. `wmill sync pull` /// passes `true` to surface ACLs in the git-tracked yaml. @@ -365,8 +365,8 @@ pub(crate) struct ArchiveQueryParams { /// pre-existing serialization for folders and groups so /// no customer sees a one-time noisy diff on upgrade. /// * `KeepIfNonEmpty` — keep when there is at least one entry, drop when `{}` -/// or null. New surface for flow / script / app, which -/// never carried ACLs in source before this change. +/// or null. New surface for script / flow / app / variable, +/// which never carried ACLs in source before this change. #[derive(Clone, Copy)] pub enum ExtraPermsBehavior { Drop, @@ -665,7 +665,7 @@ pub(crate) async fn tarball_workspace( check_scopes(&authed, || "variables:read".to_string())?; } - // Opt-in behavior for surfacing per-resource ACLs on flow/app rows. + // Opt-in behavior for surfacing per-resource ACLs on script/flow/app/variable rows. // Folder and group rows have always carried `extra_perms` in source and // continue to do so unconditionally (`KeepEvenEmpty`) so existing // customer git repos see no one-time noisy diff. @@ -1002,8 +1002,7 @@ pub(crate) async fn tarball_workspace( Error::internal_err(format!("Error decrypting variable {}: {}", var.path, e)) })?); } - let var_str = - &to_string_without_metadata(&var, ExtraPermsBehavior::Drop, None).unwrap(); + let var_str = &to_string_without_metadata(&var, new_kinds_extra_perms, None).unwrap(); archive .write_to_archive(&var_str, &format!("{}.variable.json", var.path)) .await?; diff --git a/cli/src/commands/sync/pull.ts b/cli/src/commands/sync/pull.ts index 2702e13714..68fd002eb5 100644 --- a/cli/src/commands/sync/pull.ts +++ b/cli/src/commands/sync/pull.ts @@ -104,7 +104,7 @@ export async function downloadZip( // from v1 the on-behalf-of address is stripped below, so the tarball sends the // `has_on_behalf_of` marker instead and never resolves an address. // `preserve_extra_perms=true` opts the tarball into surfacing granular ACLs - // on flow / script / app rows. Default-off on the server protects cross- + // on script / flow / app / variable rows. Default-off on the server protects cross- // workspace tarball imports from carrying ACLs that reference identities // missing in the target workspace; the CLI sync flow explicitly wants them. const baseParams = `&plain_secret=${plainSecrets ?? false diff --git a/cli/src/commands/variable/variable.ts b/cli/src/commands/variable/variable.ts index 2d885c9dd5..8963f376d2 100644 --- a/cli/src/commands/variable/variable.ts +++ b/cli/src/commands/variable/variable.ts @@ -19,6 +19,7 @@ import { sep as SEP } from "node:path"; import * as wmill from "../../../gen/services.gen.ts"; import { ListableVariable } from "../../../gen/types.gen.ts"; +import { applyExtraPermsDiff } from "../../core/extra_perms.ts"; async function list(opts: GlobalOptions & { json?: boolean }) { if (opts.json) log.setSilent(true); @@ -97,6 +98,7 @@ export interface VariableFile { description: string; account?: number; is_oauth?: boolean; + extra_perms?: Record; } /** @@ -152,37 +154,42 @@ export async function pushVariable( log.debug(`Variable ${remotePath} does not exist on remote`); } + // extra_perms is synced independently via /acls/* (see applyExtraPermsDiff) + // so a perm-only edit never rewrites the variable value. Strip the field from + // the body that goes to update_variable / create_variable and treat it as a + // separate step both for the up-to-date short-circuit and after the write. + const { extra_perms: localPerms, ...localVariableBody } = localVariable; + if (variable) { - if (isSuperset(localVariable, variable)) { + if (isSuperset(localVariableBody, variable)) { log.debug(`Variable ${remotePath} is up-to-date`); - return; - } + } else { + log.debug(`Variable ${remotePath} is not up-to-date, updating`); - log.debug(`Variable ${remotePath} is not up-to-date, updating`); - - // Apply is_secret only when it differs from the remote (the value is always - // sent, so the server allows the flag change). Upgrades (non-secret->secret) - // always apply; downgrades only when explicitly allowed (single-file push) — - // see allowSecretDowngrade. `undefined` leaves the flag untouched. - let nextIsSecret: boolean | undefined = undefined; - if (localVariable.is_secret !== variable.is_secret) { - if (localVariable.is_secret) { - nextIsSecret = true; - } else if (allowSecretDowngrade) { - nextIsSecret = false; + // Apply is_secret only when it differs from the remote (the value is always + // sent, so the server allows the flag change). Upgrades (non-secret->secret) + // always apply; downgrades only when explicitly allowed (single-file push) — + // see allowSecretDowngrade. `undefined` leaves the flag untouched. + let nextIsSecret: boolean | undefined = undefined; + if (localVariableBody.is_secret !== variable.is_secret) { + if (localVariableBody.is_secret) { + nextIsSecret = true; + } else if (allowSecretDowngrade) { + nextIsSecret = false; + } } - } - await wmill.updateVariable({ - workspace, - path: remotePath.replaceAll(SEP, "/"), - alreadyEncrypted: !plainSecrets, - requestBody: { - ...localVariable, - is_secret: nextIsSecret, - ...(wsSpecific !== undefined ? { ws_specific: wsSpecific } : {}), - }, - }); + await wmill.updateVariable({ + workspace, + path: remotePath.replaceAll(SEP, "/"), + alreadyEncrypted: !plainSecrets, + requestBody: { + ...localVariableBody, + is_secret: nextIsSecret, + ...(wsSpecific !== undefined ? { ws_specific: wsSpecific } : {}), + }, + }); + } } else { log.info(colors.yellow.bold(`Creating new variable ${remotePath}...`)); await wmill.createVariable({ @@ -190,11 +197,22 @@ export async function pushVariable( alreadyEncrypted: !plainSecrets, requestBody: { path: remotePath.replaceAll(SEP, "/"), - ...localVariable, + ...localVariableBody, ...(wsSpecific !== undefined ? { ws_specific: wsSpecific } : {}), }, }); } + + // Synced whether or not the body changed. No refetch: folder perms are never + // merged onto item.extra_perms, and the update/create body carries no + // extra_perms, so the value getVariable read above is still the remote one. + await applyExtraPermsDiff( + workspace, + "variable", + remotePath.replaceAll(SEP, "/"), + localPerms, + (variable as any)?.extra_perms, + ); } async function push( diff --git a/cli/test/variable_resource_push.test.ts b/cli/test/variable_resource_push.test.ts index e6bc84c326..b77ed48b50 100644 --- a/cli/test/variable_resource_push.test.ts +++ b/cli/test/variable_resource_push.test.ts @@ -361,6 +361,126 @@ describe("variable", () => { expect(content).toContain("is_secret: false"); }); }); + + test("extra_perms round-trips and pushes via /acls/* without rewriting the variable", async () => { + await withTestBackend(async (backend, tempDir) => { + await setupWorkspaceProfile(backend); + + const uniqueId = Date.now(); + const varPath = `f/test/perms_var_${uniqueId}`; + + const createResp = await backend.apiRequest!( + `/api/w/${backend.workspace}/variables/create`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + path: varPath, + value: "perms_test_value", + is_secret: false, + description: "Variable for extra_perms test", + }), + } + ); + expect(createResp.status).toBeLessThan(300); + await createResp.text(); + + const aclResp = await backend.apiRequest!( + `/api/w/${backend.workspace}/acls/add/variable/${varPath}`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ owner: "g/all", write: true }), + } + ); + expect(aclResp.status).toBeLessThan(300); + await aclResp.text(); + + await writeFile( + join(tempDir, "wmill.yaml"), + `defaultTs: bun\nincludes:\n - "${varPath}**"\nexcludes: []\n`, + "utf-8" + ); + + const pullResult = await backend.runCLICommand( + ["sync", "pull", "--yes"], + tempDir + ); + expect(pullResult.code).toEqual(0); + + const localPath = join(tempDir, `${varPath}.variable.yaml`); + const pulled = await readFile(localPath, "utf-8"); + expect(pulled).toContain("extra_perms:"); + expect(pulled).toContain("g/all: true"); + + const beforeResp = await backend.apiRequest!( + `/api/w/${backend.workspace}/variables/get/${varPath}` + ); + const before = await beforeResp.json(); + + // Perm-only edit: downgrade the grant to read. + await writeFile( + localPath, + pulled.replace("g/all: true", "g/all: false"), + "utf-8" + ); + + const pushResult = await backend.runCLICommand( + ["sync", "push", "--yes"], + tempDir + ); + expect(pushResult.code).toEqual(0); + + const afterResp = await backend.apiRequest!( + `/api/w/${backend.workspace}/variables/get/${varPath}` + ); + const after = await afterResp.json(); + expect(after.extra_perms).toEqual({ "g/all": false }); + // Routed through /acls/* rather than update_variable, so the row itself + // is untouched. + expect(after.edited_at).toEqual(before.edited_at); + expect(after.value).toEqual("perms_test_value"); + + // A yaml with no extra_perms field at all is "no opinion": a checkout + // that predates ACL sync must never revoke UI-managed grants. + await writeFile( + localPath, + `description: "Variable for extra_perms test"\nvalue: perms_test_value\nis_secret: false\n`, + "utf-8" + ); + const noOpinionResult = await backend.runCLICommand( + ["sync", "push", "--yes"], + tempDir + ); + expect(noOpinionResult.code).toEqual(0); + + const noOpinionApiResp = await backend.apiRequest!( + `/api/w/${backend.workspace}/variables/get/${varPath}` + ); + expect((await noOpinionApiResp.json()).extra_perms).toEqual({ + "g/all": false, + }); + + // An owner present remotely but absent from a *present* map is revoked — + // the one direction that can destroy a grant. + await writeFile( + localPath, + `description: "Variable for extra_perms test"\nvalue: perms_test_value\nis_secret: false\nextra_perms: {}\n`, + "utf-8" + ); + const revokeResult = await backend.runCLICommand( + ["sync", "push", "--yes"], + tempDir + ); + expect(revokeResult.code).toEqual(0); + + const finalResp = await backend.apiRequest!( + `/api/w/${backend.workspace}/variables/get/${varPath}` + ); + const final = await finalResp.json(); + expect(final.extra_perms).toEqual({}); + }); + }); }); // ============================================================================= From f381acdb37f66f5e272bc37938e69f734987d53f Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 7 Sep 2026 14:11:58 +0000 Subject: [PATCH 02/15] fix: seed runs page filter defaults through the url so they survive sync (#11005) Claude-Session: https://claude.ai/code/session_017KKZCLrTrWAqSeGjTzVtP2 Co-authored-by: Claude Opus 5 (1M context) --- .../src/lib/components/FilterSearchbar.svelte | 36 ++++++++++++++++-- frontend/src/lib/components/RunsPage.svelte | 37 +++++++++++-------- 2 files changed, 54 insertions(+), 19 deletions(-) diff --git a/frontend/src/lib/components/FilterSearchbar.svelte b/frontend/src/lib/components/FilterSearchbar.svelte index 51026f10fd..6c89f1a039 100644 --- a/frontend/src/lib/components/FilterSearchbar.svelte +++ b/frontend/src/lib/components/FilterSearchbar.svelte @@ -112,17 +112,37 @@ } /** - * Creates a URL-synced filter instance that automatically syncs with URL search parameters + * Creates a URL-synced filter instance that automatically syncs with URL search parameters. + * + * `initial` supplies defaults for keys the URL doesn't carry (a route segment, a persisted + * toggle); the returned `seed` re-applies them after a navigation has rewritten the query. */ export function useUrlSyncedFilterInstance( - schemaRec: T - ): { val: Partial> } { + schemaRec: T, + initial?: Partial> + ): { + val: Partial> + seed: (values: Partial>) => void + } { // Build the Zod schema from the filter schema const zodSchema = filterSchemaRecToZodSchema(schemaRec) // Create URL-synced search params const urlFilter = useSearchParams(zodSchema) as Record + // A default has to arrive as a URL param: the URL→instance effect below drops whatever the + // URL lacks, so a value written to the instance is undone on the next sync. Going through + // urlFilter rather than straight to history keeps the search-param cells in step, so it + // does not matter whether a popstate follows. + function seed(values: Partial>) { + const sp = new URLSearchParams(window.location.search) + for (const [key, value] of Object.entries(values) as [string, unknown][]) { + if (value === undefined || value === null || sp.has(key)) continue + urlFilter[key] = value instanceof Date ? value.toISOString() : value + } + } + if (initial) seed(initial) + // Create the filter instance object const filterInstance: { val: Partial> } = $state({ val: {} }) @@ -173,7 +193,15 @@ }) } - return filterInstance + return { + get val() { + return filterInstance.val + }, + set val(v: Partial>) { + filterInstance.val = v + }, + seed + } } function filterToText(filter: FilterInstance, schema: F): string { diff --git a/frontend/src/lib/components/RunsPage.svelte b/frontend/src/lib/components/RunsPage.svelte index 03b410d232..cac8952706 100644 --- a/frontend/src/lib/components/RunsPage.svelte +++ b/frontend/src/lib/components/RunsPage.svelte @@ -84,6 +84,8 @@ initialPath?: string } + let { initialPath }: Props = $props() + let paths: string[] = $state([]) let usernames: string[] = $state([]) let folders: string[] = $state([]) @@ -100,25 +102,30 @@ let perPage = useLocalStorageValue('runs_per_page', 1000, 'number') let showSchedulesStorage = useLocalStorageValue('runs_show_schedules', true, 'boolean') let showFutureJobsStorage = useLocalStorageValue('runs_show_future_jobs', true, 'boolean') - let filters = useUrlSyncedFilterInstance(untrack(() => runsFilterSearchbarSchema)) + function filterSeeds() { + return { + path: initialPath || undefined, + job_trigger_kind: showSchedulesStorage.val === false ? ('!schedule' as const) : undefined, + show_future_jobs: showFutureJobsStorage.val === false ? false : undefined + } + } - let { initialPath }: Props = $props() + let filters = useUrlSyncedFilterInstance( + untrack(() => runsFilterSearchbarSchema), + untrack(filterSeeds) + ) + + // `runs/[...path]` is a single route, so a navigation between its URLs — the sidebar's own + // "Runs" entry, `/runs/` → `/runs`, Back — rewrites the query without remounting, and + // what was seeded at mount is gone. Re-apply it. Editing a filter writes with `replaceState`, + // which never reaches `page.url`, so a filter the user clears stays cleared. + $effect(() => { + page.url.href + untrack(() => filters.seed(filterSeeds())) + }) let batchRerunOptionsIsOpen = $state(false) - // Initialize path filter from route param if provided and not already set via query params - if (untrack(() => initialPath) && !filters.val.path) { - filters.val.path = untrack(() => initialPath) - } - - // Apply persistent toggle values from local storage if URL doesn't specify them - if (!page.url.searchParams.has('job_trigger_kind') && showSchedulesStorage.val === false) { - filters.val.job_trigger_kind = '!schedule' - } - if (!page.url.searchParams.has('show_future_jobs') && showFutureJobsStorage.val === false) { - filters.val.show_future_jobs = false - } - // Sync toggle state back to local storage when filters change $effect(() => { if (!filters.val.job_trigger_kind || filters.val.job_trigger_kind === '!schedule') { From 7643e9bd77c56f72596b8dca50801baf58984198 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 7 Sep 2026 14:18:39 +0000 Subject: [PATCH 03/15] fix(cli): say which workspace id is targeted, and when wmill.yaml is bypassed (#11006) * fix(cli): say which workspace id is targeted and when wmill.yaml is bypassed Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01QaQ3UtkbHA6pxqQStqRQQj * fix(cli): make the wmill.yaml lookup for diagnostics side-effect free Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01QaQ3UtkbHA6pxqQStqRQQj * fix(cli): only report a wmill.yaml mapping that sets an explicit workspaceId Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01QaQ3UtkbHA6pxqQStqRQQj --------- Co-authored-by: Claude Opus 5 (1M context) --- cli/src/commands/sync/pull.ts | 13 +++- cli/src/core/conf.ts | 42 +++++++++++- cli/src/core/context.ts | 54 ++++++++++++++- ...base_url_workspace_resolution_unit.test.ts | 68 +++++++++++++++++++ 4 files changed, 173 insertions(+), 4 deletions(-) create mode 100644 cli/test/base_url_workspace_resolution_unit.test.ts diff --git a/cli/src/commands/sync/pull.ts b/cli/src/commands/sync/pull.ts index 68fd002eb5..d75dc81912 100644 --- a/cli/src/commands/sync/pull.ts +++ b/cli/src/commands/sync/pull.ts @@ -150,7 +150,18 @@ export async function downloadZip( } if (zipResponse.status === 404 || body.includes("no rows returned")) { - log.info(colors.red(`Workspace '${workspace.workspaceId}' not found on ${workspace.remote}. Please check your --workspace and try again.`)); + log.info( + colors.red( + `Workspace id '${workspace.workspaceId}' not found on ${workspace.remote}` + + (workspace.name !== workspace.workspaceId + ? ` (resolved from profile '${workspace.name}')` + : "") + + `.\n` + + `Note this is the workspace *id* sent to the API, which is not necessarily what you passed to --workspace:\n` + + ` - check 'wmill workspace list' (the 'workspace id' column)\n` + + ` - check the 'workspaces' block of wmill.yaml ('workspaceId' overrides the workspace name)` + ) + ); } else { log.info(colors.red(`Failed to request tarball from API: ${zipResponse.status} ${zipResponse.statusText}`)); if (body) log.info(colors.red(body)); diff --git a/cli/src/core/conf.ts b/cli/src/core/conf.ts index 1fcce7a43f..786ef48cc2 100644 --- a/cli/src/core/conf.ts +++ b/cli/src/core/conf.ts @@ -145,7 +145,9 @@ function getGitRepoRoot(): string | null { } export const GLOBAL_CONFIG_OPT = { noCdToRoot: false }; -function findWmillYaml(): string | null { + +// Pure upward search: no chdir, no logging. findWmillYaml() adds the chdir. +function locateWmillYaml(): string | null { const startDir = resolve(process.cwd()); const isInGitRepo = isGitRepository(); const gitRoot = isInGitRepo ? getGitRepoRoot() : null; @@ -176,6 +178,13 @@ function findWmillYaml(): string | null { currentDir = parentDir; } + return foundPath; +} + +function findWmillYaml(): string | null { + const startDir = resolve(process.cwd()); + const foundPath = locateWmillYaml(); + // If wmill.yaml was found in a parent directory, warn the user and change working directory if ( !GLOBAL_CONFIG_OPT.noCdToRoot && @@ -198,6 +207,37 @@ export function getWmillYamlPath(): string | null { return findWmillYaml(); } +/** + * Look up one `workspaces` entry, for diagnostics only. readConfigFile() must + * not be used for that: it chdirs to the config's directory, exits on an + * unsupported syncBehavior and throws on a malformed file. A diagnostic may + * never fail or relocate the command it is diagnosing. + */ +export async function peekWorkspaceEntry( + workspaceName: string +): Promise { + if (RESERVED_WORKSPACE_KEYS.has(workspaceName)) { + return undefined; + } + const wmillYamlPath = locateWmillYaml(); + if (!wmillYamlPath) { + return undefined; + } + try { + const conf = (await yamlParseFile(wmillYamlPath)) as SyncOptions; + const workspaces = + conf?.workspaces ?? + conf?.gitBranches ?? + conf?.environments ?? + conf?.git_branches; + const entry = (workspaces as any)?.[workspaceName]; + return typeof entry === "object" && entry !== null ? entry : undefined; + } catch (e) { + log.debug(`Failed to parse ${wmillYamlPath} for workspace lookup: ${e}`); + return undefined; + } +} + let legacyConfigWarned = false; export async function readConfigFile(opts?: { warnIfMissing?: boolean }): Promise { diff --git a/cli/src/core/context.ts b/cli/src/core/context.ts index 502157f0a7..80b27ea52e 100644 --- a/cli/src/core/context.ts +++ b/cli/src/core/context.ts @@ -20,6 +20,7 @@ import { import { getLastUsedProfile, setLastUsedProfile } from "./branch-profiles.ts"; import { readConfigFile, + peekWorkspaceEntry, findWorkspaceByGitBranch, getEffectiveWorkspaceId, getWmillYamlPath, @@ -219,6 +220,9 @@ async function tryResolveWorkspace( // First try: look up workspace by name in wmill.yaml workspaces config const config = await readConfigFile({ warnIfMissing: false }); const wsEntry = config.workspaces?.[opts.workspace] as WorkspaceEntryConfig | undefined; + // What wmill.yaml said to target, kept for the fallback below: a profile + // found by name can silently point somewhere else entirely. + let configuredTarget: { workspaceId: string; baseUrl: string } | undefined; if (wsEntry?.baseUrl) { const workspaceId = getEffectiveWorkspaceId(opts.workspace, wsEntry); let normalizedBaseUrl: string; @@ -231,6 +235,8 @@ async function tryResolveWorkspace( }; } + configuredTarget = { workspaceId, baseUrl: normalizedBaseUrl }; + // Find matching profile by baseUrl + workspaceId const allProfs = await allWorkspaces(opts.configDir); const matching = allProfs.filter( @@ -283,6 +289,22 @@ async function tryResolveWorkspace( ), }; } + if ( + configuredTarget && + (e.workspaceId !== configuredTarget.workspaceId || + e.remote !== configuredTarget.baseUrl) + ) { + log.warnStderr( + colors.yellow( + `⚠️ Falling back to the local profile named '${opts.workspace}' (${e.workspaceId} on ${e.remote}), which does NOT match wmill.yaml:\n` + + ` wmill.yaml maps workspace '${opts.workspace}' to ${configuredTarget.workspaceId} on ${configuredTarget.baseUrl}, but no profile targets it.\n` + + ` Run: wmill workspace add ${configuredTarget.workspaceId} ${configuredTarget.baseUrl}` + ) + ); + } + log.infoStderr( + `Using local profile '${e.name}' → ${e.workspaceId} on ${e.remote}` + ); (opts as any).__secret_workspace = e; return { isError: false, value: e }; } @@ -486,6 +508,8 @@ export async function resolveWorkspace( return process.exit(-1); } + let resolved: Workspace | undefined; + // Try to find existing workspace profile by name, then by workspaceId + remote if (opts.workspace) { let existingWorkspace = await getWorkspaceByName( @@ -523,19 +547,45 @@ export async function resolveWorkspace( ); return process.exit(-1); } - return { + resolved = { ...existingWorkspace, token: opts.token, }; } } - return { + resolved ??= { remote: normalizedBaseUrl, workspaceId: opts.workspace, name: opts.workspace, token: opts.token, }; + + // --base-url pins the target, so wmill.yaml's `workspaces` block is never + // consulted and `--workspace` reaches the API as a workspace id. Name the + // id being sent, and the mapping being skipped, before the request 404s + // on an id the user never typed. + // Only an explicit `workspaceId:` is worth reporting: an entry without one + // maps the name to itself, leaving nothing to correct. + const yamlEntry = await peekWorkspaceEntry(opts.workspace); + const yamlWorkspaceId = yamlEntry?.workspaceId; + if (yamlWorkspaceId && yamlWorkspaceId !== resolved.workspaceId) { + log.warnStderr( + colors.yellow( + `⚠️ --base-url is set, so wmill.yaml is not consulted: workspace id '${resolved.workspaceId}' is sent to the API.\n` + + ` wmill.yaml maps workspace '${opts.workspace}' to workspace id '${yamlWorkspaceId}'${yamlEntry!.baseUrl ? ` on ${yamlEntry!.baseUrl}` : ""}.\n` + + ` Use '--workspace ${yamlWorkspaceId}', or drop --base-url/--token to resolve through wmill.yaml.` + ) + ); + } + log.infoStderr( + `Using workspace id '${resolved.workspaceId}' on ${normalizedBaseUrl} (--base-url given` + + (resolved.name !== resolved.workspaceId + ? `, profile '${resolved.name}')` + : ")") + ); + (opts as any).__secret_workspace = resolved; + return resolved; } else { log.infoStderr( colors.red( diff --git a/cli/test/base_url_workspace_resolution_unit.test.ts b/cli/test/base_url_workspace_resolution_unit.test.ts new file mode 100644 index 0000000000..66f618d113 --- /dev/null +++ b/cli/test/base_url_workspace_resolution_unit.test.ts @@ -0,0 +1,68 @@ +import { describe, expect, test } from "bun:test"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; + +import { resolveWorkspace } from "../src/core/context.ts"; +import { getWorkspaceConfigFilePath } from "../windmill-utils-internal/src/config/config.ts"; +import type { GlobalOptions } from "../src/types.ts"; + +const BASE_URL = "http://localhost:9999/"; + +// --base-url pins the target: --workspace reaches the API as a workspace id and +// wmill.yaml is not consulted. The warning that says so may only peek at the +// file — readConfigFile() exits on an unsupported syncBehavior and throws on a +// malformed one, so resolving through it lets an unrelated config fail a +// command that never needed it. +async function withWmillYaml( + wmillYaml: string, + fn: (opts: GlobalOptions) => Promise +): Promise { + const repoDir = await mkdtemp(path.join(os.tmpdir(), "wmill_baseurl_repo_")); + const configDir = await mkdtemp(path.join(os.tmpdir(), "wmill_baseurl_conf_")); + const originalCwd = process.cwd(); + try { + await writeFile(path.join(repoDir, "wmill.yaml"), wmillYaml); + await writeFile(await getWorkspaceConfigFilePath(configDir), ""); + + process.chdir(repoDir); + await fn({ + configDir, + baseUrl: BASE_URL, + token: "sometoken", + workspace: "staging", + } as GlobalOptions); + } finally { + process.chdir(originalCwd); + await rm(repoDir, { recursive: true, force: true }); + await rm(configDir, { recursive: true, force: true }); + } +} + +describe("--base-url workspace resolution", () => { + const rejectedConfigs: [string, string][] = [ + ["an unsupported syncBehavior", "syncBehavior: v2\n"], + ["a malformed file", 'workspaces:\n staging:\n baseUrl: "unterminated\n'], + ]; + + for (const [label, wmillYaml] of rejectedConfigs) { + test(`resolves despite ${label}`, async () => { + await withWmillYaml(wmillYaml, async (opts) => { + const workspace = await resolveWorkspace(opts); + expect(workspace.workspaceId).toBe("staging"); + expect(workspace.remote).toBe(BASE_URL); + }); + }); + } + + test("a workspaces mapping never overrides the explicit workspace id", async () => { + await withWmillYaml( + "workspaces:\n staging:\n baseUrl: http://elsewhere.example/\n workspaceId: admins\n", + async (opts) => { + const workspace = await resolveWorkspace(opts); + expect(workspace.workspaceId).toBe("staging"); + expect(workspace.remote).toBe(BASE_URL); + } + ); + }); +}); From 5da4ea43fbd01e43aa14e75dc597d7ce5d8797ab Mon Sep 17 00:00:00 2001 From: AlexRV12 <71396855+AlexRV12@users.noreply.github.com> Date: Mon, 7 Sep 2026 16:22:09 +0200 Subject: [PATCH 04/15] feat: show the new-tab icon on a chat path pill while the modifier is held (#10976) * feat: show the new-tab icon on a chat path pill while the modifier is held Fixes WIN-2477 Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01LAa9nNcYxDN3qAZPrYg4Lq * fix: read the new-tab modifier in the capture phase Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01LAa9nNcYxDN3qAZPrYg4Lq * refactor: track the new-tab modifier only while a pill is hovered The window key listeners were installed at import time and never removed, so every page that loaded the module paid for them whether or not a pill existed. They now attach on mouseenter and detach on mouseleave or destroy, which is the only window in which the answer is read. Seeding the flag from the hover event also removes the limitation the previous version documented: a mouse event carries the same modifier flags as a key event, so a modifier held before the pointer arrived, or while this window was unfocused, now reads correctly. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01LAa9nNcYxDN3qAZPrYg4Lq * refactor: export the new-tab modifier as a read-only view `newTabModifier` handed every consumer a writable handle on module-global state, so any of them could drive the icon of every pill on the page. The getter form is what frontend/AGENTS.md prescribes for shared reactive state. Tearing each attachment down in the test's afterEach as well: the module state and its window listeners outlive the DOM, so emptying the body left `held` and the hovered node set for the following case. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01LAa9nNcYxDN3qAZPrYg4Lq * refactor: only track the modifier for pills whose icon can change The attachment went on every path pill, so hovering a drawer or plain-link pill installed three window listeners for a flag its icon never reads. Only a preview pill can flip, so only it gets them. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01LAa9nNcYxDN3qAZPrYg4Lq * fix: re-read the new-tab modifier from pointer movement A modifier held across a keyboard app switch was cleared by the blur and never restored: the key was down the whole time so no keydown arrived on the way back, and the pointer parked on the pill fired no fresh mouseenter either. The pill then showed the panel icon while the click would have opened a tab. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01LAa9nNcYxDN3qAZPrYg4Lq * refactor: give each pill its own modifier state The shared module state forced a node-identity guard: one hovered element owned the window listeners, so a pill destroyed elsewhere in the transcript had to be stopped from tearing them down. A factory per pill removes the guard, its test case, and the whole class of cross-instance interference, and narrows re-renders to the hovered pill instead of every preview pill on screen. Listener teardown now goes through AbortController signals, so leaving a pill drops the whole set at once rather than through a remove list that has to mirror every option exactly. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01LAa9nNcYxDN3qAZPrYg4Lq * fix: abort the previous hover controller on re-entry A second mouseenter with no mouseleave between replaced the controller without aborting it, so the four listeners registered under the first signal outlived even the element's destruction: neither leave nor the destroy path held a reference to reach them. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01LAa9nNcYxDN3qAZPrYg4Lq --------- Co-authored-by: Claude Opus 5 (1M context) --- .../attachments/newTabModifier.dom.test.ts | 106 ++++++++++++++++++ .../lib/attachments/newTabModifier.svelte.ts | 66 +++++++++++ .../copilot/chat/LinkRenderer.svelte | 13 ++- 3 files changed, 183 insertions(+), 2 deletions(-) create mode 100644 frontend/src/lib/attachments/newTabModifier.dom.test.ts create mode 100644 frontend/src/lib/attachments/newTabModifier.svelte.ts diff --git a/frontend/src/lib/attachments/newTabModifier.dom.test.ts b/frontend/src/lib/attachments/newTabModifier.dom.test.ts new file mode 100644 index 0000000000..115877298c --- /dev/null +++ b/frontend/src/lib/attachments/newTabModifier.dom.test.ts @@ -0,0 +1,106 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { newTabModifier } from './newTabModifier.svelte' + +const onPlatform = (userAgent: string) => vi.stubGlobal('navigator', { userAgent }) +const LINUX = 'Mozilla/5.0 (X11; Linux x86_64)' +const MAC = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)' + +const attached: (() => void)[] = [] + +/** Attach to a fresh element and return it with its cleanup, as `{@attach}` would. */ +function pill() { + const node = document.createElement('span') + document.body.append(node) + const modifier = newTabModifier() + const cleanup = modifier.attach(node) as () => void + attached.push(cleanup) + const hover = (init: MouseEventInit = {}) => + node.dispatchEvent(new MouseEvent('mouseenter', init)) + const move = (init: MouseEventInit = {}) => node.dispatchEvent(new MouseEvent('mousemove', init)) + const unhover = () => node.dispatchEvent(new MouseEvent('mouseleave')) + return { modifier, hover, move, unhover, cleanup } +} + +const keydown = (init: KeyboardEventInit) => + window.dispatchEvent(new KeyboardEvent('keydown', init)) + +describe('newTabModifier', () => { + // The window listeners outlive the DOM, so every case has to be torn down through the + // attachment rather than by emptying the body. + afterEach(() => { + attached.splice(0).forEach((cleanup) => cleanup()) + document.body.replaceChildren() + vi.unstubAllGlobals() + }) + + // The hover event carries the live modifier state, so a modifier pressed before the pointer + // arrived (or while this window was unfocused) is picked up rather than read as false. + it('seeds from the hover event, per platform', () => { + onPlatform(LINUX) + const linux = pill() + linux.hover({ ctrlKey: true }) + expect(linux.modifier.held).toBe(true) + + onPlatform(MAC) + const mac = pill() + // macOS ctrl+click is a secondary click, so it must not read as a new-tab modifier. + mac.hover({ ctrlKey: true }) + expect(mac.modifier.held).toBe(false) + mac.hover({ metaKey: true }) + expect(mac.modifier.held).toBe(true) + }) + + // Editors and menus stop keydown propagation to keep their own shortcuts, so a bubble-phase + // listener would go blind whenever focus sits in one. + it('sees a keydown that a focused element stops from propagating', () => { + onPlatform(LINUX) + const { modifier, hover } = pill() + hover() + const input = document.createElement('input') + input.addEventListener('keydown', (e) => e.stopPropagation()) + document.body.append(input) + + input.dispatchEvent( + new KeyboardEvent('keydown', { key: 'Control', ctrlKey: true, bubbles: true }) + ) + expect(modifier.held).toBe(true) + }) + + // A modifier held across a keyboard app switch is cleared by the blur and delivers no keydown + // on the way back, while the pointer parked on the pill fires no fresh mouseenter either. + it('re-seeds from pointer movement after the window lost focus', () => { + onPlatform(LINUX) + const { modifier, hover, move } = pill() + hover({ ctrlKey: true }) + window.dispatchEvent(new Event('blur')) + expect(modifier.held).toBe(false) + + move({ ctrlKey: true }) + expect(modifier.held).toBe(true) + }) + + it('stops tracking once unhovered', () => { + onPlatform(LINUX) + const { modifier, hover, unhover } = pill() + hover({ ctrlKey: true }) + unhover() + expect(modifier.held).toBe(false) + + keydown({ key: 'Control', ctrlKey: true }) + expect(modifier.held).toBe(false) + }) + + it('stops tracking when the element is destroyed while hovered', () => { + onPlatform(LINUX) + const { modifier, hover, cleanup } = pill() + hover({ ctrlKey: true }) + // Hovering again without leaving must not strand the first hover's listeners, which nothing + // would then hold a reference to. + hover({ ctrlKey: true }) + cleanup() + expect(modifier.held).toBe(false) + + keydown({ key: 'Control', ctrlKey: true }) + expect(modifier.held).toBe(false) + }) +}) diff --git a/frontend/src/lib/attachments/newTabModifier.svelte.ts b/frontend/src/lib/attachments/newTabModifier.svelte.ts new file mode 100644 index 0000000000..06c69f36cc --- /dev/null +++ b/frontend/src/lib/attachments/newTabModifier.svelte.ts @@ -0,0 +1,66 @@ +import type { Attachment } from 'svelte/attachments' +import { isMac } from '$lib/utils' + +/** + * Tracks whether the modifier that turns a click into a new browser tab is held, but only while + * the attached element is hovered, which is the only moment the answer is used. + */ +export function newTabModifier() { + let held = $state(false) + + // Only the modifier that actually yields a tab: shift opens a window, alt can start a + // download, and on macOS ctrl+click is a secondary click. + // Taken from each event rather than accumulated across keydown/keyup pairs, so a keyup lost to + // a focus change cannot strand the flag on. + const sync = (event: KeyboardEvent | MouseEvent) => { + held = isMac() ? event.metaKey : event.ctrlKey + } + const clear = () => { + held = false + } + + const attach: Attachment = (node) => { + // One controller per hover: a mirrored remove list leaks any listener whose options drift. + let hover: AbortController | undefined + const leave = () => { + hover?.abort() + hover = undefined + clear() + } + const enter = (event: MouseEvent) => { + // Seeded from the hover itself: mouse events carry the same modifier flags as key events, + // so a modifier already held before the pointer arrived reads correctly. + sync(event) + // Re-entering without an intervening leave would strand the previous controller: nothing + // else references it, so its listeners could never be removed. + hover?.abort() + hover = new AbortController() + const { signal } = hover + // Same reason the hover seeds: a modifier held across a keyboard app switch delivers no + // keydown on the way back, so the pointer is all that is left to re-read it from. + node.addEventListener('mousemove', sync, { signal }) + // Capture: editors and menus stopPropagation the keys they handle, hiding the modifier + // from a bubble-phase listener whenever focus sits in one. + window.addEventListener('keydown', sync, { capture: true, signal }) + window.addEventListener('keyup', sync, { capture: true, signal }) + // Not capture, unlike the two above: blur does not bubble but does reach the window while + // capturing, so it would fire for every element that loses focus. + window.addEventListener('blur', clear, { signal }) + } + + const life = new AbortController() + node.addEventListener('mouseenter', enter, { signal: life.signal }) + node.addEventListener('mouseleave', leave, { signal: life.signal }) + return () => { + life.abort() + leave() + } + } + + return { + get held() { + return held + }, + attach + } +} diff --git a/frontend/src/lib/components/copilot/chat/LinkRenderer.svelte b/frontend/src/lib/components/copilot/chat/LinkRenderer.svelte index 1e885fa304..e93017b095 100644 --- a/frontend/src/lib/components/copilot/chat/LinkRenderer.svelte +++ b/frontend/src/lib/components/copilot/chat/LinkRenderer.svelte @@ -3,6 +3,7 @@ import { ExternalLink, PanelRight } from 'lucide-svelte' import { Button } from '$lib/components/common' import RowIcon from '$lib/components/common/table/RowIcon.svelte' + import { newTabModifier } from '$lib/attachments/newTabModifier.svelte' import { hasToolDisplayActionHandler, runToolDisplayAction @@ -44,6 +45,8 @@ const previewAction = $derived(available?.type === 'open_item_preview' ? available : undefined) const drawerAction = $derived(available?.type === 'open_created_resource' ? available : undefined) + const modifier = newTabModifier() + const hint = $derived( previewAction ? `Open ${wmPath} in the preview panel` : `Open ${wmPath} in a new tab` ) @@ -67,7 +70,11 @@ {#if href} {#if wmKind} - + + - {#if previewAction} + + {#if previewAction && !modifier.held} {:else} From e2b63d177ae4e5c980cb5da34154540c90771b63 Mon Sep 17 00:00:00 2001 From: Diego Imbert <70353967+diegoimbert@users.noreply.github.com> Date: Mon, 7 Sep 2026 16:46:00 +0200 Subject: [PATCH 05/15] feat: go to referenced row from foreign-keyed cells in the database manager (#10998) * feat: go to referenced row from foreign-keyed cells in the database manager Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_0157Kw1ukbQo7eZnmyM63G4t * fix: pin foreign keys to their table and escape backslashes on snowflake Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_0157Kw1ukbQo7eZnmyM63G4t * fix: address review on foreign key navigation (stale fetch, qualifiers, chip) Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_0157Kw1ukbQo7eZnmyM63G4t * fix: unicode literals on sql server and hide unreachable foreign key targets Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_0157Kw1ukbQo7eZnmyM63G4t --------- Co-authored-by: Claude Fable 5.1 --- frontend/src/lib/components/DBManager.svelte | 107 ++++++++++-- .../lib/components/DBManagerContent.svelte | 3 +- frontend/src/lib/components/DBTable.svelte | 160 +++++++++++++++--- .../lib/components/DbForeignKeyTooltip.svelte | 23 +++ .../display/dbtable/renderDbLiteral.test.ts | 48 ++++++ .../apps/components/display/dbtable/utils.ts | 51 +++++- frontend/src/lib/components/dbOps.ts | 84 +++++---- 7 files changed, 394 insertions(+), 82 deletions(-) create mode 100644 frontend/src/lib/components/DbForeignKeyTooltip.svelte create mode 100644 frontend/src/lib/components/apps/components/display/dbtable/renderDbLiteral.test.ts diff --git a/frontend/src/lib/components/DBManager.svelte b/frontend/src/lib/components/DBManager.svelte index 35bd88abec..7520cc5013 100644 --- a/frontend/src/lib/components/DBManager.svelte +++ b/frontend/src/lib/components/DBManager.svelte @@ -12,8 +12,8 @@ import { Pane, Splitpanes } from 'svelte-splitpanes' import { ClearableInput, Drawer, DrawerContent } from './common' import { sendUserToast } from '$lib/toast' - import { type ColumnDef } from './apps/components/display/dbtable/utils' - import DBTable from './DBTable.svelte' + import { renderDbEqualityFilter, type ColumnDef } from './apps/components/display/dbtable/utils' + import DBTable, { type DbForeignKeyTarget, type DbRowFilter } from './DBTable.svelte' import type { IDbSchemaOps, IDbTableOps } from './dbOps' import DropdownV2 from './DropdownV2.svelte' import ConfirmationModal from './common/confirmationModal/ConfirmationModal.svelte' @@ -46,7 +46,12 @@ dbSupportsSchemas: boolean databaseIsEmpty?: boolean colDefs: Record | undefined - dbTableOpsFactory: (params: { colDefs: ColumnDef[]; tableKey: string }) => IDbTableOps + dbTableOpsFactory: (params: { + colDefs: ColumnDef[] + tableKey: string + /** Raw SQL predicate AND-ed into the reads (already escaped). */ + whereClause?: string + }) => IDbTableOps dbSchemaOps: IDbSchemaOps refresh?: () => void initialSchemaKey?: string @@ -214,6 +219,83 @@ : selected.tableKey ) + // Set by "Go to row" on a foreign-keyed cell; pinned to the table it was + // created for so a schema change can't carry it onto an unrelated table. + let rowFilter: (DbRowFilter & { tableKey: string }) | undefined = $state() + let activeRowFilter = $derived(rowFilter?.tableKey === tableKey ? rowFilter : undefined) + let whereClause = $derived( + activeRowFilter + ? renderDbEqualityFilter(activeRowFilter.column, activeRowFilter.value, dbType) + : undefined + ) + + function selectTable(schemaKey: string | undefined, table: string) { + rowFilter = undefined + selected = { schemaKey, tableKey: table } + } + + /** Where a foreign key's `schema.table` target lives in the sidebar, or + * undefined when it cannot be opened from here. */ + function resolveForeignKeyTarget( + targetTable: string + ): { schemaKey: string; table: string } | undefined { + const parts = targetTable.split('.') + const table = parts[parts.length - 1] + const qualifier = parts.length > 1 ? parts.slice(0, -1).join('.') : undefined + // Without schema support the sidebar browses the connection's default + // schema only, and unqualified reads would hit a same-named local table. + if (!dbSupportsSchemas && qualifier && qualifier !== selected.schemaKey) return undefined + const schemaKey = dbSupportsSchemas && qualifier ? qualifier : selected.schemaKey + if (!schemaKey || !(table in (dbSchema.schema[schemaKey] ?? {}))) return undefined + return { schemaKey, table } + } + + function goToRow(target: DbForeignKeyTarget) { + const resolved = resolveForeignKeyTarget(target.table) + if (!resolved) { + sendUserToast(`Table ${target.table} cannot be opened from this schema`, true) + return + } + if (renderDbEqualityFilter(target.column, target.value, dbType) === undefined) { + sendUserToast('This value cannot be used as a filter', true) + return + } + const { schemaKey, table } = resolved + selectTable(schemaKey, table) + rowFilter = { + tableKey: dbSupportsSchemas ? `${schemaKey}.${table}` : table, + column: target.column, + value: target.value + } + } + + // The result carries the table it was fetched for: `resource` keeps the + // previous value while refetching, and a stale list would decorate the new + // table's same-named columns as foreign keys. + let foreignKeys = resource( + [() => selected.tableKey, () => selected.schemaKey, () => colDefs], + async ([table, schema], _prev, { signal }) => { + if (!table) return undefined + const forTableKey = dbSupportsSchemas && schema ? `${schema}.${table}` : table + const fks = + features?.foreignKeys === false + ? [] + : await dbSchemaOps.onFetchForeignKeys({ table, schema }) + // A newer selection started meanwhile: an AbortError keeps this result + // out of `current`, where it would shadow the newer table's keys. + if (signal.aborted) throw new DOMException('Superseded', 'AbortError') + return { tableKey: forTableKey, foreignKeys: fks } + } + ) + // Only keys whose target the sidebar can open get the "Go to row" affordance. + let currentForeignKeys = $derived.by(() => { + const fetched = foreignKeys.current + if (!fetched || fetched.tableKey !== tableKey) return undefined + return fetched.foreignKeys.filter( + (fk) => fk.targetTable && resolveForeignKeyTarget(fk.targetTable) !== undefined + ) + }) + let askingForConfirmation: | (ConfirmationModal['$$prop_def'] & { onConfirm: () => void }) | undefined = $state() @@ -395,14 +477,12 @@ role="button" tabindex="0" onclick={() => { - selected.schemaKey = schemaKey - selected.tableKey = tableKey + selectTable(schemaKey, tableKey) toggleTableSelection(schemaKey, tableKey) }} onkeydown={(e) => { if (e.key === 'Enter' || e.key === ' ') { - selected.schemaKey = schemaKey - selected.tableKey = tableKey + selectTable(schemaKey, tableKey) toggleTableSelection(schemaKey, tableKey) } }} @@ -468,7 +548,7 @@ + diff --git a/frontend/src/lib/components/apps/components/display/dbtable/renderDbLiteral.test.ts b/frontend/src/lib/components/apps/components/display/dbtable/renderDbLiteral.test.ts new file mode 100644 index 0000000000..1dec5e9da8 --- /dev/null +++ b/frontend/src/lib/components/apps/components/display/dbtable/renderDbLiteral.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from 'vitest' +import { renderDbEqualityFilter, renderDbLiteral } from './utils' + +describe('renderDbLiteral', () => { + it('doubles single quotes on every dialect', () => { + expect(renderDbLiteral("O'Brien", 'postgresql')).toBe("'O''Brien'") + expect(renderDbLiteral("O'Brien", 'mysql')).toBe("'O''Brien'") + }) + + it('doubles backslashes only where the dialect treats them as escapes', () => { + expect(renderDbLiteral('C:\\dir\\', 'postgresql')).toBe("'C:\\dir\\'") + expect(renderDbLiteral('C:\\dir\\', 'mysql')).toBe("'C:\\\\dir\\\\'") + expect(renderDbLiteral('C:\\dir\\', 'snowflake')).toBe("'C:\\\\dir\\\\'") + }) + + it('marks SQL Server strings as Unicode constants', () => { + expect(renderDbLiteral("Zoë's", 'ms_sql_server')).toBe("N'Zoë''s'") + }) + + it('renders numbers and booleans without quotes', () => { + expect(renderDbLiteral(42, 'postgresql')).toBe('42') + expect(renderDbLiteral(true, 'postgresql')).toBe('TRUE') + expect(renderDbLiteral(true, 'ms_sql_server')).toBe('1') + }) + + it('has no literal for values that cannot be compared safely', () => { + expect(renderDbLiteral(null, 'postgresql')).toBeUndefined() + expect(renderDbLiteral({ a: 1 }, 'postgresql')).toBeUndefined() + expect(renderDbLiteral(NaN, 'postgresql')).toBeUndefined() + }) +}) + +describe('renderDbEqualityFilter', () => { + it('quotes the identifier per dialect', () => { + expect(renderDbEqualityFilter('user id', 'x', 'postgresql')).toBe(`"user id" = 'x'`) + expect(renderDbEqualityFilter('user id', 'x', 'ms_sql_server')).toBe(`[user id] = N'x'`) + expect(renderDbEqualityFilter('user id', 'x', 'mysql')).toBe("`user id` = 'x'") + expect(renderDbEqualityFilter('user id', null, 'postgresql')).toBeUndefined() + }) + + it('doubles a delimiter embedded in the identifier', () => { + expect(renderDbEqualityFilter('a"b', 1, 'postgresql')).toBe(`"a""b" = 1`) + expect(renderDbEqualityFilter('a"b', 1, 'snowflake')).toBe(`"a""b" = 1`) + expect(renderDbEqualityFilter('a"b', 1, 'duckdb')).toBe(`"a""b" = 1`) + expect(renderDbEqualityFilter('a]b', 1, 'ms_sql_server')).toBe(`[a]]b] = 1`) + expect(renderDbEqualityFilter('a`b', 1, 'mysql')).toBe('`a``b` = 1') + }) +}) diff --git a/frontend/src/lib/components/apps/components/display/dbtable/utils.ts b/frontend/src/lib/components/apps/components/display/dbtable/utils.ts index bf81ec07bb..c0477e8408 100644 --- a/frontend/src/lib/components/apps/components/display/dbtable/utils.ts +++ b/frontend/src/lib/components/apps/components/display/dbtable/utils.ts @@ -333,25 +333,58 @@ export function duckdbQuicksearchColumns(columnDefs: ColumnDef[]): string { .join(', ') } +/** Mirrors the backend's `render_db_quoted_identifier`, including doubling an + * embedded delimiter. */ export function renderDbQuotedIdentifier(identifier: string, dbType: DbType): string { switch (dbType) { case 'postgresql': - return `"${identifier}"` // PostgreSQL uses double quotes for identifiers - case 'ms_sql_server': - return `[${identifier}]` // MSSQL uses square brackets for identifiers - case 'mysql': - return `\`${identifier}\`` // MySQL uses backticks case 'snowflake': - return `"${identifier}"` // Snowflake uses double quotes for identifiers - case 'bigquery': - return `\`${identifier}\`` // BigQuery uses backticks case 'duckdb': - return `"${identifier}"` // DuckDB uses double quotes for identifiers + return `"${identifier.replace(/"/g, '""')}"` + case 'ms_sql_server': + return `[${identifier.replace(/]/g, ']]')}]` + case 'mysql': + case 'bigquery': + return `\`${identifier.replace(/`/g, '``')}\`` default: throw new Error('Unsupported database type: ' + dbType) } } +/** Renders a cell value as a SQL literal. Returns undefined for values that + * have no safe literal form (null, objects, non-finite numbers). */ +export function renderDbLiteral(value: unknown, dbType: DbType): string | undefined { + if (value === null || value === undefined) return undefined + if (typeof value === 'number') return Number.isFinite(value) ? String(value) : undefined + if (typeof value === 'bigint') return value.toString() + if (typeof value === 'boolean') { + if (dbType === 'ms_sql_server') return value ? '1' : '0' + return value ? 'TRUE' : 'FALSE' + } + if (typeof value !== 'string') return undefined + let escaped = value.replace(/'/g, "''") + // MySQL, Snowflake and BigQuery treat a backslash inside a string literal as + // an escape character. + if (dbType === 'mysql' || dbType === 'snowflake' || dbType === 'bigquery') { + escaped = escaped.replace(/\\/g, '\\\\') + } + // A plain constant is varchar on SQL Server and goes through the database + // code page; the N prefix keeps it Unicode against nvarchar columns. + return dbType === 'ms_sql_server' ? `N'${escaped}'` : `'${escaped}'` +} + +/** `"column" = ` predicate, or undefined when the value can't be + * rendered as a literal. */ +export function renderDbEqualityFilter( + column: string, + value: unknown, + dbType: DbType +): string | undefined { + const literal = renderDbLiteral(value, dbType) + if (literal === undefined) return undefined + return `${renderDbQuotedIdentifier(column, dbType)} = ${literal}` +} + export function getLanguageByResourceType(name: string): ScriptLang { const language = { postgresql: 'postgresql', diff --git a/frontend/src/lib/components/dbOps.ts b/frontend/src/lib/components/dbOps.ts index a89abf4374..5005cc931c 100644 --- a/frontend/src/lib/components/dbOps.ts +++ b/frontend/src/lib/components/dbOps.ts @@ -14,6 +14,7 @@ import { WorkspaceService } from '$lib/gen' import { pendingMigrations } from './workspaceSettings/datatableMigrationUtils' import { buildTableEditorValues, + type TableEditorForeignKey, type TableEditorValues } from './apps/components/display/dbtable/tableEditor' import { type AlterTableValues } from './apps/components/display/dbtable/queries/alterTable' @@ -250,6 +251,10 @@ export type IDbSchemaOps = { schema?: string colDefs: TableMetadata }) => Promise + onFetchForeignKeys: (params: { + table: string + schema?: string + }) => Promise } /** Thrown by a schema op when the user declines the out-of-order run warning. @@ -396,6 +401,48 @@ export function dbSchemaOpsWithPreviewScripts({ } } + /** Resolves to [] when the database has no foreign key introspection + * (BigQuery) or the query fails: callers treat foreign keys as optional. */ + async function fetchForeignKeys({ + table, + schema + }: { + table: string + schema?: string + }): Promise { + if (dbType === 'bigquery') return [] + try { + const fkContent = makeMarker('FOREIGN_KEYS', { table, schema }) + const fkResult = await runScriptAndPollResult({ + workspace, + requestBody: { args: dbArg, content: fkContent, language, tag } + }) + + let rawForeignKeys: RawForeignKey[] + if (dbType === 'snowflake') { + rawForeignKeys = transformSnowflakeForeignKeys(fkResult as any[]) + } else { + rawForeignKeys = fkResult as RawForeignKey[] + if (rawForeignKeys && Array.isArray(rawForeignKeys)) { + rawForeignKeys = rawForeignKeys.map((fk) => { + const lowerFk: any = {} + Object.keys(fk).forEach((key) => { + lowerFk[key.toLowerCase()] = fk[key] + }) + return lowerFk + }) + } + } + + if (rawForeignKeys && Array.isArray(rawForeignKeys)) { + return transformForeignKeys(rawForeignKeys) + } + } catch (e) { + console.warn('Failed to fetch foreign keys:', e) + } + return [] + } + return { onDelete: async ({ tableKey, schema }) => { const content = makeMarker('DROP_TABLE', { table: tableKey, schema }) @@ -454,44 +501,11 @@ export function dbSchemaOpsWithPreviewScripts({ const downContent = makeMarker('CREATE_SCHEMA', { schema }) await applyDdl(migrationName('drop_schema', schema), content, downContent) }, + onFetchForeignKeys: fetchForeignKeys, onFetchTableEditorDefinition: async ({ table, schema, colDefs }) => { - let foreignKeys: import('./apps/components/display/dbtable/tableEditor').TableEditorForeignKey[] = - [] + const foreignKeys = await fetchForeignKeys({ table, schema }) let pk_constraint_name: string | undefined - // Fetch foreign keys (not supported for BigQuery) - if (dbType !== 'bigquery') { - try { - const fkContent = makeMarker('FOREIGN_KEYS', { table, schema }) - const fkResult = await runScriptAndPollResult({ - workspace, - requestBody: { args: dbArg, content: fkContent, language, tag } - }) - - let rawForeignKeys: RawForeignKey[] - if (dbType === 'snowflake') { - rawForeignKeys = transformSnowflakeForeignKeys(fkResult as any[]) - } else { - rawForeignKeys = fkResult as RawForeignKey[] - if (rawForeignKeys && Array.isArray(rawForeignKeys)) { - rawForeignKeys = rawForeignKeys.map((fk) => { - const lowerFk: any = {} - Object.keys(fk).forEach((key) => { - lowerFk[key.toLowerCase()] = fk[key] - }) - return lowerFk - }) - } - } - - if (rawForeignKeys && Array.isArray(rawForeignKeys)) { - foreignKeys = transformForeignKeys(rawForeignKeys) - } - } catch (e) { - console.warn('Failed to fetch foreign keys:', e) - } - } - // Fetch primary key constraint name (not supported for BigQuery/MySQL) if (dbType !== 'bigquery' && dbType !== 'mysql') { try { From 5f3f99ba6915b7c5df663a30b35f4cd02050e728 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Mon, 7 Sep 2026 16:46:35 +0200 Subject: [PATCH 06/15] fix(cli): keep permissioned_as on single-item push, as sync push does (#11000) * fix(cli): keep permissioned_as on single-item push, as sync push does * fix(cli): resolve syncBehavior from the target workspace, not the branch alone * refactor(cli): share the workspace-name resolution between sync and single-item push * test(cli): import the moved workspace-name helper from its new home --- cli/src/commands/app/app.ts | 20 ++- cli/src/commands/flow/flow.ts | 21 ++- cli/src/commands/schedule/schedule.ts | 19 ++- cli/src/commands/script/script.ts | 8 +- cli/src/commands/sync/sync.ts | 88 ++---------- cli/src/commands/trigger/trigger.ts | 11 +- cli/src/core/conf.ts | 77 +++++++++++ cli/src/core/permissioned_as.ts | 30 +++++ ...schedule_push_permissioned_as_unit.test.ts | 125 ++++++++++++++++++ ...workspace_key_filename_integration.test.ts | 6 +- 10 files changed, 316 insertions(+), 89 deletions(-) create mode 100644 cli/test/schedule_push_permissioned_as_unit.test.ts diff --git a/cli/src/commands/app/app.ts b/cli/src/commands/app/app.ts index 0f46ec76be..b35efc02a3 100644 --- a/cli/src/commands/app/app.ts +++ b/cli/src/commands/app/app.ts @@ -12,7 +12,11 @@ import * as wmill from "../../../gen/services.gen.ts"; import { ListableApp, Policy } from "../../../gen/types.gen.ts"; import { GlobalOptions, isSuperset } from "../../types.ts"; -import { getWmillYamlPath, mergeConfigWithConfigFile } from "../../core/conf.ts"; +import { + getWmillYamlPath, + mergeConfigWithConfigFile, + readEffectiveSyncBehavior, +} from "../../core/conf.ts"; import { readInlinePathSync } from "../../utils/utils.ts"; import devCommand from "./dev.ts"; import lintCommand from "./lint.ts"; @@ -21,6 +25,7 @@ import newCommand from "./new.ts"; import generateAgentsCommand from "./generate_agents.ts"; import { isVersionsGeq1585 } from "../sync/global.ts"; import type { PermissionedAsContext } from "../../core/permissioned_as.ts"; +import { buildPermissionedAsContext } from "../../core/permissioned_as.ts"; import { applyExtraPermsDiff } from "../../core/extra_perms.ts"; export interface AppFile { @@ -420,6 +425,8 @@ async function push( if (isRawAppByName || hasRawAppYaml) { const { pushRawApp } = await import("./raw_apps.ts"); const merged = await mergeConfigWithConfigFile(opts); + // Raw-app ownership preservation is not implemented on either push + // path: sync push hands pushRawApp no context either. await pushRawApp( workspace.workspaceId, remotePath, @@ -429,7 +436,16 @@ async function push( ); log.info(colors.bold.underline.green("Raw app pushed")); } else { - await pushApp(workspace.workspaceId, remotePath, absoluteFilePath); + await pushApp( + workspace.workspaceId, + remotePath, + absoluteFilePath, + undefined, + await buildPermissionedAsContext( + workspace.workspaceId, + await readEffectiveSyncBehavior(opts, workspace), + ), + ); log.info(colors.bold.underline.green("App pushed")); } } diff --git a/cli/src/commands/flow/flow.ts b/cli/src/commands/flow/flow.ts index 7a383fcfeb..863690ce7d 100644 --- a/cli/src/commands/flow/flow.ts +++ b/cli/src/commands/flow/flow.ts @@ -4,7 +4,7 @@ import { Command } from "@cliffy/command"; import { Confirm } from "@cliffy/prompt/confirm"; import { Table } from "@cliffy/table"; import * as log from "../../core/log.ts"; -import { dirname, sep as SEP } from "node:path"; +import { dirname, sep as SEP, resolve as pathResolve } from "node:path"; import { stringify as yamlStringify } from "yaml"; import { yamlParseFile } from "../../utils/yaml.ts"; import { readTextFile, validateRequiredArgs } from "../../utils/utils.ts"; @@ -21,11 +21,16 @@ import { } from "../../core/context.ts"; import { resolve, track_job, pollForJobResult } from "../script/script.ts"; import { defaultFlowDefinition } from "../../../bootstrap/flow_bootstrap.ts"; -import { SyncOptions, mergeConfigWithConfigFile } from "../../core/conf.ts"; +import { + SyncOptions, + mergeConfigWithConfigFile, + readEffectiveSyncBehavior, +} from "../../core/conf.ts"; import { FSFSElement, elementsToMap, ignoreF } from "../sync/sync.ts"; import { Flow } from "../../../gen/types.gen.ts"; import { applyExtraPermsDiff } from "../../core/extra_perms.ts"; import type { PermissionedAsContext } from "../../core/permissioned_as.ts"; +import { buildPermissionedAsContext } from "../../core/permissioned_as.ts"; import { collectPathScriptPaths, replaceInlineScripts, @@ -327,10 +332,20 @@ async function push(opts: Options & { message?: string }, filePath: string, remo if (!validatePath(remotePath)) { return; } + // Reading the config moves the cwd to the wmill.yaml root when it sits in a + // parent directory, so pin the file against the invocation cwd first. + filePath = pathResolve(filePath); const workspace = await resolveWorkspace(opts); await requireLogin(opts); + const syncBehavior = await readEffectiveSyncBehavior(opts, workspace); - await pushFlow(workspace.workspaceId, remotePath, filePath, opts.message); + await pushFlow( + workspace.workspaceId, + remotePath, + filePath, + opts.message, + await buildPermissionedAsContext(workspace.workspaceId, syncBehavior) + ); log.info(colors.bold.underline.green("Flow pushed")); } diff --git a/cli/src/commands/schedule/schedule.ts b/cli/src/commands/schedule/schedule.ts index 2eb6ba42b8..096615c757 100644 --- a/cli/src/commands/schedule/schedule.ts +++ b/cli/src/commands/schedule/schedule.ts @@ -6,13 +6,19 @@ import { Command } from "@cliffy/command"; import { Table } from "@cliffy/table"; import { colors } from "@cliffy/ansi/colors"; import * as log from "../../core/log.ts"; -import { sep as SEP } from "node:path"; +import { sep as SEP, resolve as pathResolve } from "node:path"; import { requireLogin } from "../../core/auth.ts"; import { resolveWorkspace, validatePath } from "../../core/context.ts"; -import { mergeConfigWithConfigFile } from "../../core/conf.ts"; +import { + mergeConfigWithConfigFile, + readEffectiveSyncBehavior, +} from "../../core/conf.ts"; import * as wmill from "../../../gen/services.gen.ts"; import type { PermissionedAsContext } from "../../core/permissioned_as.ts"; -import { lookupUsernameByEmail } from "../../core/permissioned_as.ts"; +import { + buildPermissionedAsContext, + lookupUsernameByEmail, +} from "../../core/permissioned_as.ts"; import { GlobalOptions, @@ -299,8 +305,12 @@ async function disable(opts: GlobalOptions, path: string) { } async function push(opts: GlobalOptions, filePath: string, remotePath: string) { + // Reading the config moves the cwd to the wmill.yaml root when it sits in a + // parent directory, so pin the file against the invocation cwd first. + filePath = pathResolve(filePath); const workspace = await resolveWorkspace(opts); await requireLogin(opts); + const syncBehavior = await readEffectiveSyncBehavior(opts, workspace); if (!validatePath(remotePath)) { return; @@ -317,7 +327,8 @@ async function push(opts: GlobalOptions, filePath: string, remotePath: string) { workspace.workspaceId, remotePath, undefined, - parseFromFile(filePath) + parseFromFile(filePath), + await buildPermissionedAsContext(workspace.workspaceId, syncBehavior) ); console.log(colors.bold.underline.green("Schedule pushed")); } diff --git a/cli/src/commands/script/script.ts b/cli/src/commands/script/script.ts index fada434e0d..d2afdeeee3 100644 --- a/cli/src/commands/script/script.ts +++ b/cli/src/commands/script/script.ts @@ -7,6 +7,7 @@ import { validatePath, } from "../../core/context.ts"; import type { PermissionedAsContext } from "../../core/permissioned_as.ts"; +import { buildPermissionedAsContext } from "../../core/permissioned_as.ts"; import { applyExtraPermsDiff } from "../../core/extra_perms.ts"; import { writeFile, stat, mkdir } from "node:fs/promises"; import { Buffer } from "node:buffer"; @@ -58,6 +59,7 @@ import { SyncOptions, mergeConfigWithConfigFile, readConfigFile, + readEffectiveSyncBehavior, } from "../../core/conf.ts"; import { SyncCodebase, listSyncCodebases } from "../../utils/codebase.ts"; import { pollJobWithQueueLogging } from "../../utils/job_polling.ts"; @@ -231,7 +233,11 @@ async function push(opts: PushOptions, filePath: string) { opts.message, opts, await getRawWorkspaceDependencies(true), - codebases + codebases, + await buildPermissionedAsContext( + workspace.workspaceId, + await readEffectiveSyncBehavior(opts, workspace) + ) ); log.info(colors.bold.underline.green(`Script ${filePath} pushed`)); } diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index be5d9e5639..99a098f848 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -76,7 +76,8 @@ import { } from "../../utils/utils.ts"; import { getEffectiveSettings, - getWorkspaceNames, + inferWsNameFromProfile, + resolveWsNameForConfigFromFlags, mergeConfigWithConfigFile, parseSyncBehavior, SyncOptions, @@ -85,7 +86,10 @@ import { WorkspaceEntryConfig, } from "../../core/conf.ts"; import type { PermissionedAsContext } from "../../core/permissioned_as.ts"; -import { preCheckPermissionedAs } from "../../core/permissioned_as.ts"; +import { + buildPermissionedAsContext, + preCheckPermissionedAs, +} from "../../core/permissioned_as.ts"; import { fromWorkspaceSpecificPath, toWorkspaceSpecificPath, @@ -429,37 +433,6 @@ export function computeWsSpecificFlagOnlyPushes( return out; } -// Resolve workspace name from a --branch override (git branch → workspace name). -// Falls back to using the branch value as-is (backward compat: old key = branch name). -function resolveWsNameFromBranch( - opts: SyncOptions, - branchName: string, -): string { - const match = findWorkspaceByGitBranch(opts.workspaces, branchName); - return match ? match[0] : branchName; -} - -// Resolve wsNameForConfig from CLI flags. Prefers --branch → matching config key, -// then --workspace → matching config key (incl. when --base-url is set). Returns -// undefined when no flag-based resolution applies; callers then fall back to -// inferWsNameFromProfile on the resolved workspace profile. -export function resolveWsNameForConfigFromFlags( - opts: SyncOptions & { branch?: string; workspace?: string }, -): string | undefined { - if (opts.branch) { - return resolveWsNameFromBranch(opts, opts.branch); - } - if (opts.workspace) { - // Use getWorkspaceNames so reserved keys (e.g. commonSpecificItems) are filtered out, - // matching the behavior of findWorkspaceByGitBranch / inferWsNameFromProfile. - const validKeys = getWorkspaceNames(opts.workspaces); - if (validKeys.includes(opts.workspace)) { - return opts.workspace; - } - } - return undefined; -} - // Warn if --workspace overrides auto-detected branch or if workspace not in config. function warnWorkspaceOverride( opts: SyncOptions, @@ -507,33 +480,6 @@ function resolveWsNameForFiles(_opts: SyncOptions, wsName: string): string { return wsName; } -// After resolveWorkspace, infer the workspace config name from the resolved profile -// by matching baseUrl + workspaceId against the workspaces config entries. -function inferWsNameFromProfile( - opts: SyncOptions, - profile: { remote: string; workspaceId: string }, -): string | undefined { - if (!opts.workspaces) return undefined; - const wsNames = Object.keys(opts.workspaces).filter( - (k) => k !== "commonSpecificItems", - ); - for (const name of wsNames) { - const entry = (opts.workspaces as any)[name] as WorkspaceEntryConfig; - if (!entry?.baseUrl) continue; - try { - const entryUrl = new URL(entry.baseUrl).toString(); - const profileUrl = new URL(profile.remote).toString(); - const entryWsId = entry.workspaceId ?? name; - if (entryUrl === profileUrl && entryWsId === profile.workspaceId) { - return name; - } - } catch { - continue; - } - } - return undefined; -} - // Merge CLI options with effective settings, preserving CLI flags as overrides function mergeCliWithEffectiveOptions< T extends GlobalOptions & SyncOptions & { repository?: string }, @@ -5540,27 +5486,19 @@ export async function push( return; } - let permissionedAsContext: PermissionedAsContext | undefined = undefined; - if (parseSyncBehavior(opts.syncBehavior) >= 1) { - const user = await wmill.whoami({ workspace: workspace.workspaceId }); - const userIsAdminOrDeployer = - user.is_admin || (user.groups ?? []).includes("wm_deployers"); - log.debug( - `permissioned_as: user=${user.email}, is_admin=${user.is_admin}, groups=${JSON.stringify(user.groups)}, isAdminOrDeployer=${userIsAdminOrDeployer}`, + const permissionedAsContext: PermissionedAsContext | undefined = + await buildPermissionedAsContext( + workspace.workspaceId, + opts.syncBehavior, ); - permissionedAsContext = { - userCache: new Map(), - userIsAdminOrDeployer, - userEmail: user.email, - }; - + if (permissionedAsContext) { // ws_specific_flag changes have no content payload, so they don't // affect permissioned_as resolution — filter them out before the // pre-check (which expects only added/edited/deleted). await preCheckPermissionedAs( changes.filter((c) => c.name !== "ws_specific_flag"), - user.email, - userIsAdminOrDeployer, + permissionedAsContext.userEmail, + permissionedAsContext.userIsAdminOrDeployer, opts.acceptOverridingPermissionedAsWithSelf ?? false, !!process.stdin.isTTY, ); diff --git a/cli/src/commands/trigger/trigger.ts b/cli/src/commands/trigger/trigger.ts index 83b62f9eb4..b011a0f0b7 100644 --- a/cli/src/commands/trigger/trigger.ts +++ b/cli/src/commands/trigger/trigger.ts @@ -23,7 +23,7 @@ import { Command } from "@cliffy/command"; import { Table } from "@cliffy/table"; import { colors } from "@cliffy/ansi/colors"; import * as log from "../../core/log.ts"; -import { sep as SEP } from "node:path"; +import { sep as SEP, resolve as pathResolve } from "node:path"; import { GlobalOptions, isSuperset, @@ -41,6 +41,8 @@ import { getCurrentGitBranch } from "../../utils/git.ts"; import { requireLogin } from "../../core/auth.ts"; import { validatePath, resolveWorkspace } from "../../core/context.ts"; import type { PermissionedAsContext } from "../../core/permissioned_as.ts"; +import { buildPermissionedAsContext } from "../../core/permissioned_as.ts"; +import { readEffectiveSyncBehavior } from "../../core/conf.ts"; type Trigger = { http: HttpTrigger; @@ -620,8 +622,12 @@ async function extractTriggerKindFromPath(filePath: string): Promise.overrides.syncBehavior`, which is where a repo + * that varies settings per workspace puts it, and the entry to read is the one + * `--workspace` names — falling back to the profile, then to the git branch — + * the same order `sync push` resolves it in. + */ +export async function readEffectiveSyncBehavior( + opts: { workspace?: string }, + profile?: { remote: string; workspaceId: string } +): Promise { + const config = await readConfigFile({ warnIfMissing: false }); + const named = resolveWsNameForConfigFromFlags({ ...config, ...opts }); + const effective = await getEffectiveSettings( + config, + undefined, + false, + true, + named ?? (profile ? inferWsNameFromProfile(config, profile) : undefined) + ); + return effective.syncBehavior; +} + const RESERVED_WORKSPACE_KEYS = new Set(["commonSpecificItems"]); /** diff --git a/cli/src/core/permissioned_as.ts b/cli/src/core/permissioned_as.ts index 5ac48753d6..57570b3549 100644 --- a/cli/src/core/permissioned_as.ts +++ b/cli/src/core/permissioned_as.ts @@ -3,6 +3,7 @@ import * as log from "./log.ts"; import { colors } from "@cliffy/ansi/colors"; import { Confirm } from "@cliffy/prompt/confirm"; import { getTypeStrFromPath } from "../types.ts"; +import { parseSyncBehavior } from "./conf.ts"; export interface PermissionedAsContext { userCache: Map; @@ -10,6 +11,35 @@ export interface PermissionedAsContext { userEmail: string; } +/** + * The whole-tree `sync push` and the single-item `push` commands must resolve + * ownership the same way, so both build the context here: a push that leaves it + * undefined reassigns `permissioned_as` / `on_behalf_of` to whoever ran it. + * Undefined below syncBehavior v1, where that reassignment is the contract, and + * for a caller who is neither admin nor in `wm_deployers` the backend enforces + * it anyway — the flag on the context is what keeps the CLI from claiming + * otherwise. + */ +export async function buildPermissionedAsContext( + workspace: string, + syncBehavior: string | number | undefined +): Promise { + if (parseSyncBehavior(syncBehavior) < 1) { + return undefined; + } + const user = await wmill.whoami({ workspace }); + const userIsAdminOrDeployer = + user.is_admin || (user.groups ?? []).includes("wm_deployers"); + log.debug( + `permissioned_as: user=${user.email}, is_admin=${user.is_admin}, groups=${JSON.stringify(user.groups)}, isAdminOrDeployer=${userIsAdminOrDeployer}` + ); + return { + userCache: new Map(), + userIsAdminOrDeployer, + userEmail: user.email, + }; +} + async function ensureUserCache( workspace: string, cache: Map diff --git a/cli/test/schedule_push_permissioned_as_unit.test.ts b/cli/test/schedule_push_permissioned_as_unit.test.ts new file mode 100644 index 0000000000..941415409d --- /dev/null +++ b/cli/test/schedule_push_permissioned_as_unit.test.ts @@ -0,0 +1,125 @@ +/** + * Regression guard: the standalone `wmill schedule push` must resolve ownership + * the same way `wmill sync push` does. It only preserves the remote's + * `permissioned_as` when the command hands `pushSchedule` a context, so a push + * that builds none silently reassigns the schedule to whoever ran it. + */ + +import { expect, test, describe, beforeEach, mock } from "bun:test"; +import { mkdtemp, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +let updateScheduleCalls: any[] = []; +let remotePermissionedAs: string | undefined = "u/svc"; + +const REMOTE_SCHEDULE = () => ({ + path: "u/admin/sched", + schedule: "0 0 */6 * * *", + timezone: "Etc/UTC", + script_path: "u/admin/script", + is_flow: false, + args: {}, + enabled: false, + summary: "before", + permissioned_as: remotePermissionedAs, +}); + +mock.module("../gen/services.gen.ts", () => ({ + getSchedule: async () => REMOTE_SCHEDULE(), + updateSchedule: async (a: unknown) => { + updateScheduleCalls.push(a); + }, + whoami: async () => ({ + email: "deployer@windmill.dev", + username: "deployer", + is_admin: true, + groups: [], + }), +})); + +const realContext = await import("../src/core/context.ts"); +mock.module("../src/core/context.ts", () => ({ + ...realContext, + resolveWorkspace: async () => ({ + workspaceId: "w", + name: "w", + remote: "http://localhost/", + token: "t", + }), +})); + +const realAuth = await import("../src/core/auth.ts"); +mock.module("../src/core/auth.ts", () => ({ + ...realAuth, + requireLogin: async () => ({}), +})); + +const scheduleCommand = (await import("../src/commands/schedule/schedule.ts")) + .default; + +async function pushIn(wmillYamlTail: string): Promise { + const dir = await mkdtemp(join(tmpdir(), "windmill_sched_push_")); + await writeFile( + join(dir, "wmill.yaml"), + `defaultTs: bun\nincludeSchedules: true\n${wmillYamlTail}`, + "utf-8" + ); + await writeFile( + join(dir, "sched.schedule.yaml"), + `schedule: "0 0 */6 * * *"\ntimezone: Etc/UTC\nscript_path: u/admin/script\nis_flow: false\nargs: {}\nenabled: false\nsummary: after\n`, + "utf-8" + ); + + const cwd = process.cwd(); + process.chdir(dir); + try { + await scheduleCommand.parse([ + "push", + "sched.schedule.yaml", + "u/admin/sched", + ]); + } finally { + process.chdir(cwd); + } +} + +describe("wmill schedule push ownership", () => { + beforeEach(() => { + updateScheduleCalls = []; + remotePermissionedAs = "u/svc"; + }); + + test("keeps the remote's permissioned_as under syncBehavior v1", async () => { + await pushIn("syncBehavior: v1\n"); + + expect(updateScheduleCalls).toHaveLength(1); + const body = updateScheduleCalls[0].requestBody; + expect(body.summary).toBe("after"); + expect(body.permissioned_as).toBe("u/svc"); + expect(body.preserve_permissioned_as).toBe(true); + }); + + // The entry to read is the one matching the workspace being pushed to, not + // the top level: a repo that varies settings per workspace puts syncBehavior + // under `overrides` and nowhere else. + test("reads syncBehavior from the target workspace's overrides", async () => { + await pushIn( + `workspaces:\n other:\n baseUrl: http://localhost/\n workspaceId: w\n overrides:\n syncBehavior: v1\n` + ); + + expect(updateScheduleCalls).toHaveLength(1); + const body = updateScheduleCalls[0].requestBody; + expect(body.permissioned_as).toBe("u/svc"); + expect(body.preserve_permissioned_as).toBe(true); + }); + + test("leaves ownership to the backend below syncBehavior v1", async () => { + await pushIn(""); + + expect(updateScheduleCalls).toHaveLength(1); + const body = updateScheduleCalls[0].requestBody; + expect(body.permissioned_as).toBeUndefined(); + expect(body.preserve_permissioned_as).toBeUndefined(); + }); +}); diff --git a/cli/test/workspace_key_filename_integration.test.ts b/cli/test/workspace_key_filename_integration.test.ts index ae22f191bf..a2e731eae5 100644 --- a/cli/test/workspace_key_filename_integration.test.ts +++ b/cli/test/workspace_key_filename_integration.test.ts @@ -7,8 +7,10 @@ import { stringify as yamlStringify } from "yaml"; import { resolveWsNameForGitBranch } from "../src/core/specific_items.ts"; import { findResourceFile } from "../src/commands/script/script.ts"; -import { resolveWsNameForConfigFromFlags } from "../src/commands/sync/sync.ts"; -import type { SyncOptions } from "../src/core/conf.ts"; +import { + resolveWsNameForConfigFromFlags, + type SyncOptions, +} from "../src/core/conf.ts"; // Integration tests covering the bug where workspace-specific filenames used // the raw git branch name instead of the wmill.yaml workspace config key. From c6e0302d7c1c60147f19d55a3923be8b1aa99c9c Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 7 Sep 2026 15:38:07 +0000 Subject: [PATCH 07/15] feat: let `// materialize` declare a `dbt://` warehouse-relation write (#10978) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: let `// materialize` declare a `dbt://` warehouse-relation write `// materialize manual dbt:////` lets an ingestion script in any language declare that it writes a warehouse relation, so it and the dbt model reading that relation land on one asset node instead of two disconnected pictures. `manual` is the only mode a warehouse target has — nothing generates warehouse DDL — and the non-`manual` spelling is refused rather than silently degraded. The `` segment is resolved against the workspace's configured warehouses, like a descriptor's `profile.warehouse`. The run records the same `materialized_partition` row a DuckLake target does, from the generic job path rather than an executor: the DuckLake write engine is DuckDB's, this declaration is anyone's. With a non-dbt producer now possible, the blanket deploy-time refusal of `# on dbt://` narrows to the shape that still cannot fire — every writer of the relation being a dbt script, since a dbt run does not dispatch. "Nothing produces it yet" stays accepted, as for every other asset kind, so deploy order does not matter. A dbt script may not subscribe at all: its graph ingest clears its own `dbt://` trigger rows. The one ordering the deploy cannot catch — a subscription accepted before any producer, then claimed by a dbt project — is named in that project's deploy log. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Rw1WrKeRRzyYHjfkuB83ek * fix: address review — preview stamping, stale producer set, public doc Three findings from the local review round: - Record the warehouse write only for a DEPLOYED script job. The annotation is a deploy-time contract (`manual`, three segments, a configured warehouse) checked where write access to the path is also required; honouring it in a preview, hub or inline-flow body let `jobs:run` alone restamp any relation's last writer from a script that never touched it. - Exclude the deploying script's own rows from the producer set. Read committed, they describe the version being replaced, so a script dropping its `// materialize` while adding a subscription counted itself as the producer that would wake it and committed a dormant edge. It could not be that producer anyway — the dispatcher skips self-loops. - `AssetKind::Dbt`'s doc no longer claims dbt is the exclusive producer of a warehouse relation, on both the types and the parser enum. Co-Authored-By: Claude Opus 5 (1M context) * fix: review round 1 — dbt-script materialize, set-form rule, doc - Refuse `// materialize` on a dbt script, the producer half of the rule the trigger loop already applies to `// on`: the graph ingest republishes that path's asset rows wholesale, so a declared write is wiped by the deploy that accepted it while its runs keep stamping the relation. - `dormant_dbt_subscriptions` now spells the same predicate its singular sibling does: the producer set has to be non-empty (nothing produces it yet is deploy order, not a dormant edge) and excludes the subscriber's own path (a script never wakes itself). Both divergences are pinned by tests. - The docs no longer claim the dbt deploy log covers a native producer that drops its `// materialize`; it does not, and nothing else reports that case. - An integration test over the deploy contract, since only a real deploy proves the handler feeds `sole_dbt_producer` the canonical key `asset.path` holds — the spelling that has to agree across the materialize target, the `// on` ref and the refusal that joins them. Co-Authored-By: Claude Opus 5 (1M context) * docs: qualify the any-language claim, and pin the dbt-script refusal `AssetKind::Dbt`'s contract (both enums), the two runtime guides and the deploy comment said a script of any language may declare a `dbt://` write, which the dbt-script refusal added last round contradicts. They now say "any language but dbt's own", with the reason: a project's writes are read from its manifest. The deploy-contract integration test covers that refusal for both annotations. Co-Authored-By: Claude Opus 5 (1M context) * docs: teach the pipeline AI guidance the warehouse-relation target The pipeline prompt (both sources, plus the regenerated bundle) told the model `// materialize` is DuckDB-only and rejected on any other target, which now steers users away from the very thing this PR adds. It distinguishes the managed DuckLake write, still DuckDB-only, from the warehouse-relation declaration any language but dbt's own may make. `dbt_manifest.rs`'s module doc carried the same "the only thing that creates one" overclaim the other four sites lost last commit. Co-Authored-By: Claude Opus 5 (1M context) * fix: draw an explicit dbt:// subscription on the canvas The editor suppressed every `// on dbt://…` overlay, which was right while the deploy refused all of them. It now refuses only a relation dbt alone builds, so the suppression hid the author's own annotation for exactly the case this PR adds — a subscription woken by a native `// materialize manual dbt://…` producer. The deploy stays the gate. Also the two stale claims round 4 named: the live pipeline prompt dropped the dbt-script exception the base prompt carries, and the doc's e2e requirements still said every `dbt://` subscription is refused. Co-Authored-By: Claude Opus 5 (1M context) * fix: refuse `// data_test` beside a `dbt://` materialize target `// data_test` checks are verifier probes the DuckDB executor splices around a managed write. A warehouse relation is written by the script itself, in any language, so nothing would run them — and unlike the DuckLake `manual` case, which at least fails loudly in that executor, a declarer in another language deployed green with its data-quality assertions silently skipped. Covered in the deploy-contract test and documented beside the annotation. Co-Authored-By: Claude Opus 5 (1M context) * fix: exclude a renamed producer from the sole-dbt producer set The producer set already excluded the deploying script's own path, because its committed rows describe the version being replaced. Under a rename the write sits at the OLD path — still committed, and removed by the same uncommitted transaction — so a producer renamed while it drops its `// materialize` and adds `// on dbt://…` still counted as the producer that would wake it, and committed a dormant edge. The deploy-contract test covers it: without the exclusion the rename deploys 201 instead of being refused. Co-Authored-By: Claude Opus 5 (1M context) * test: take the rename test's parent hash from the create response `format!("{:x}", …)` over the stored i64 drops leading zeros, while `ScriptHash`'s deserializer hex-decodes and demands 8 bytes — so a hash below 2^60 would 422 the request instead of reaching the refusal it asserts on, on roughly one in sixteen spellings of that script body. The create response already carries the zero-padded form, as the rest of the suite uses. Co-Authored-By: Claude Opus 5 (1M context) * docs: state the concurrent-ingest interleaving honestly `sole_dbt_producer`'s doc claimed the concurrent-deploy race only ever resolves toward refusing. It does when the uncommitted producer is native; when it is the dbt ingest, the check sees an empty producer set and accepts, and if that ingest then commits and runs its warning query before the subscriber's trigger row lands, neither side reports the dormant edge. Not serialized: the two would have to share a per-relation lock, and the ingest takes `script … FOR UPDATE` before its own advisory lock, so a deploy holding relation locks first inverts that order into a cross-subsystem deadlock — a worse failure than the cosmetic edge. Recorded beside the other orphaning the deploy cannot catch, with the bound both share: the next deploy of that project warns. Co-Authored-By: Claude Opus 5 (1M context) * fix: refuse a `dbt://` subscription that is not a whole relation `# on dbt://main/analytics` deployed and persisted a trigger row. Every producer spells `//` — the manifest ingest derives it from `relation_name`, a `// materialize` target is checked against it — so a partial one is an edge nothing can ever wake, which is what the dbt-only refusal exists to prevent. The shape now has one definition (`is_full_relation_path`) that both halves of the deploy ask, rather than a segment count spelled twice: a subscription and a write that disagreed would refuse and accept the same string. Also rewrites the canvas test's comment as a current constraint per AGENTS.md. Co-Authored-By: Claude Opus 5 (1M context) * fix: hold both halves of the deploy to one `dbt://` relation validator A subscription checked the relation's shape but not its warehouse, so `# on dbt:////` deployed and persisted a trigger row for something no producer can ever write: the write side refuses that exact string, and a dbt project's `profile.warehouse` resolves against the same config, so no later deploy fixes it and the dormant-edge warning cannot report it either. The shape rule and the warehouse rule now live in one `validate_dbt_relation` that both halves call, rather than being spelled per site — the previous two rounds each closed one half of one rule, which is the drift that invites. Also moves the parser test out from between a comment and the test it documents, and names both refusals in the doc's list. Co-Authored-By: Claude Opus 5 (1M context) * fix: drop the subscription-only clause from the shared refusal message "so nothing can produce it" reads backwards on the `// materialize` side, which is the producer. The remaining sentence says what is wrong on both. Co-Authored-By: Claude Opus 5 (1M context) * fix: bound a `dbt://` relation by the asset-path column in the shared validator `asset.path` is VARCHAR(255) and the manifest ingest drops a relation that outgrows it rather than failing the whole graph, so past the column no producer row can exist on either side. `script_trigger.trigger_ref` is unbounded text, so an overlong subscription deployed and stayed dormant for good; an overlong write reached Postgres and failed the deploy on a `value too long` instead of a message. Both now refuse in the validator the two halves share, against the ingest's own constant. The integration case computes the ref from that constant so it cannot drift back under the bound. Co-Authored-By: Claude Opus 5 (1M context) * fix: report a warehouse-lookup failure as the failure it is, and correct the boundary `dbt_warehouse_exists` fails three ways — no such warehouse, the query itself, and a setting with no `resource_path` — and all three became a 400 blaming the user's warehouse name. A pool timeout mid-deploy told a retrying sync that a transient server error was a permanent client one. Only `NotFound` is the annotation's fault now. The known-boundary paragraph claimed a flow-runner run still cascades. It does not: it is routed by `flow_step_id`, which `is_eligible_kind` rejects, as `asset_trigger_dispatch.rs` pins. Recording and cascading are decided separately, so the paragraph now names all three routes rather than merging two of them — and the row it omitted, an ordinary flow step, which records and never cascades. E2E item 7 said "deployable" where the rule is "wakeable": with only the dbt project reading the relation the producer set is empty, which deploys fine. Co-Authored-By: Claude Opus 5 (1M context) * docs: correct two rationales the last commit got wrong `Error::SqlErr` already maps to 400 in this codebase, so the query case's status was never the thing at stake. What the `NotFound` match earns is that a query failure and a malformed setting stop being described as an unconfigured warehouse name, and that the malformed-setting `InternalErr` reaches its own 500 instead of being flattened. And a flow step is two shapes, not one: a step running a deployed script is a `Script` job that records and never cascades, while a step with an inline body is `FlowScript`, which the recording guard excludes along with previews. Co-Authored-By: Claude Opus 5 (1M context) * fix: warn about dormant subscriptions from the run that publishes ownership too A run whose static descriptor finds its profile moved re-ingests the version's graph and republishes path ownership, exactly as a deploy does — so it can be what leaves a subscription accepted while the relation had no producer with dbt as its only one. That path discarded `persist_ingest`'s result and emitted no warning, which also made the doc's enumeration of unreported orphanings wrong. Both ownership-publishing points warn now. An agent worker still cannot: it reaches these tables only through the API and its ingest publishes without reading back, which the doc now says. Co-Authored-By: Claude Opus 5 (1M context) * docs: an agent run publishes no ownership, and the warning has two callers The agent-worker sentence called it an exception that publishes ownership without warning. It publishes none: `Connection::Http` forces per-run models, and `publishes_ownership()` is the negation of that, so an agent stores a job-pinned snapshot and leaves workspace ownership with the deployed graph — it cannot orphan a subscription at all. `warn_dormant_subscribers`' own doc still named the deploy log as the only place the warning shows, one commit after it gained its second caller. Co-Authored-By: Claude Opus 5 (1M context) * docs: stop the managed-write rule from contradicting the dbt:// target The sentence after the warehouse-relation paragraph says `// materialize` means the runtime writes the table for you and the body is a bare SELECT. That is the managed DuckLake rule, written before a `dbt://` target existed, and unqualified it tells the model the opposite of what the paragraph above it just said — a model following the more prominent one emits a SELECT for a warehouse relation, which deploys and then writes nothing. Both prompt sources now scope it, and both name the `// data_test` refusal beside a `dbt://` target, which the badge list advertised without the caveat. Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Opus 5 (1M context) --- ...84411c125cbfada861992e3d6032de635bad6.json | 12 + ...0cb1059f63c1ea51db3471c213018b2983704.json | 12 + ...7f518433741305d891d99059184eaf39fa2db.json | 20 ++ ...e613381cb4dcf7e851197150a3771ab43c99e.json | 23 ++ ...b9877190091923a2615d7d94bdf89a12e43c7.json | 12 + ...81b0a03e44beea80da732a4ad7166cf4c06cf.json | 20 ++ ...638a42f03599ce5eb872cf601f1da2e76b946.json | 63 +++++ ...3b3b6856b44d49c0712f08e307f950f210570.json | 12 + .../windmill-parser/src/asset_parser.rs | 69 +++-- .../tests/dbt_materialize_target.rs | 250 ++++++++++++++++++ backend/windmill-api-scripts/src/scripts.rs | 202 +++++++++++--- backend/windmill-common/src/assets.rs | 138 +++++++++- backend/windmill-common/src/dbt_manifest.rs | 8 +- .../tests/dbt_producer_rules.rs | 196 ++++++++++++++ backend/windmill-queue/src/asset_dispatch.rs | 12 +- backend/windmill-types/src/assets.rs | 23 +- backend/windmill-worker/src/dbt_executor.rs | 69 ++++- .../windmill-worker/src/duckdb_executor.rs | 4 +- backend/windmill-worker/src/worker.rs | 107 +++++++- docs/dbt-runtime.md | 150 ++++++++--- docs/ducklake-materialization.md | 6 +- .../assets/AssetGraph/resolveGraph.test.ts | 12 + .../assets/AssetGraph/resolveGraph.ts | 28 +- frontend/src/lib/components/assets/lib.ts | 9 +- .../components/copilot/chat/pipeline/core.ts | 4 +- system_prompts/auto-generated/prompts.ts | 8 +- system_prompts/base/pipeline-base.md | 8 +- 27 files changed, 1326 insertions(+), 151 deletions(-) create mode 100644 backend/.sqlx/query-1d8effff5dd1e4a177efed7366e84411c125cbfada861992e3d6032de635bad6.json create mode 100644 backend/.sqlx/query-1f7608defb5748db687145750520cb1059f63c1ea51db3471c213018b2983704.json create mode 100644 backend/.sqlx/query-33aa15957f56281947963cd696f7f518433741305d891d99059184eaf39fa2db.json create mode 100644 backend/.sqlx/query-49d905cae6ba42a3df62bad9385e613381cb4dcf7e851197150a3771ab43c99e.json create mode 100644 backend/.sqlx/query-6890e2be43ff8c653ce62f8cde1b9877190091923a2615d7d94bdf89a12e43c7.json create mode 100644 backend/.sqlx/query-ae4c0e8aeeef90d08a9b9c919b881b0a03e44beea80da732a4ad7166cf4c06cf.json create mode 100644 backend/.sqlx/query-b8161f6481460bed1c985bedc7c638a42f03599ce5eb872cf601f1da2e76b946.json create mode 100644 backend/.sqlx/query-e37b74f77cfa8769aee4d95155a3b3b6856b44d49c0712f08e307f950f210570.json create mode 100644 backend/windmill-api-integration-tests/tests/dbt_materialize_target.rs create mode 100644 backend/windmill-common/tests/dbt_producer_rules.rs diff --git a/backend/.sqlx/query-1d8effff5dd1e4a177efed7366e84411c125cbfada861992e3d6032de635bad6.json b/backend/.sqlx/query-1d8effff5dd1e4a177efed7366e84411c125cbfada861992e3d6032de635bad6.json new file mode 100644 index 0000000000..b60dd43fdf --- /dev/null +++ b/backend/.sqlx/query-1d8effff5dd1e4a177efed7366e84411c125cbfada861992e3d6032de635bad6.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings\n SET dbt_warehouses = '{\"main\": {\"resource_path\": \"u/test-user/wh\"}}'::jsonb\n WHERE workspace_id = 'test-workspace'", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "1d8effff5dd1e4a177efed7366e84411c125cbfada861992e3d6032de635bad6" +} diff --git a/backend/.sqlx/query-1f7608defb5748db687145750520cb1059f63c1ea51db3471c213018b2983704.json b/backend/.sqlx/query-1f7608defb5748db687145750520cb1059f63c1ea51db3471c213018b2983704.json new file mode 100644 index 0000000000..77b00d9daa --- /dev/null +++ b/backend/.sqlx/query-1f7608defb5748db687145750520cb1059f63c1ea51db3471c213018b2983704.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind)\n VALUES ('test-workspace', 'main/analytics/marts', 'dbt', 'w', 'u/test-user/project',\n 'script')", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "1f7608defb5748db687145750520cb1059f63c1ea51db3471c213018b2983704" +} diff --git a/backend/.sqlx/query-33aa15957f56281947963cd696f7f518433741305d891d99059184eaf39fa2db.json b/backend/.sqlx/query-33aa15957f56281947963cd696f7f518433741305d891d99059184eaf39fa2db.json new file mode 100644 index 0000000000..fbfab14017 --- /dev/null +++ b/backend/.sqlx/query-33aa15957f56281947963cd696f7f518433741305d891d99059184eaf39fa2db.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path FROM asset WHERE workspace_id = 'test-workspace' AND kind = 'dbt' AND usage_path = 'u/test-user/ingest' AND usage_access_type = 'w'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false + ] + }, + "hash": "33aa15957f56281947963cd696f7f518433741305d891d99059184eaf39fa2db" +} diff --git a/backend/.sqlx/query-49d905cae6ba42a3df62bad9385e613381cb4dcf7e851197150a3771ab43c99e.json b/backend/.sqlx/query-49d905cae6ba42a3df62bad9385e613381cb4dcf7e851197150a3771ab43c99e.json new file mode 100644 index 0000000000..24e563d9ad --- /dev/null +++ b/backend/.sqlx/query-49d905cae6ba42a3df62bad9385e613381cb4dcf7e851197150a3771ab43c99e.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH producer AS (\n SELECT 'dbt://' || a.path AS trigger_ref, a.usage_path, s.language\n FROM asset a\n JOIN script s ON s.workspace_id = a.workspace_id AND s.path = a.usage_path\n AND s.archived = false AND s.deleted = false\n WHERE a.workspace_id = $1 AND a.kind = 'dbt'\n AND a.usage_kind = 'script' AND a.usage_access_type IN ('w', 'rw')\n AND 'dbt://' || a.path = ANY($2)\n )\n SELECT DISTINCT st.trigger_ref || ' → ' || st.runnable_path AS \"edge!\"\n FROM script_trigger st\n WHERE st.workspace_id = $1 AND st.trigger_kind = 'asset'\n AND st.trigger_ref = ANY($2)\n AND EXISTS (SELECT 1 FROM producer p\n WHERE p.trigger_ref = st.trigger_ref\n AND p.usage_path <> st.runnable_path\n AND p.language = 'dbt')\n AND NOT EXISTS (SELECT 1 FROM producer p\n WHERE p.trigger_ref = st.trigger_ref\n AND p.usage_path <> st.runnable_path\n AND p.language <> 'dbt')\n ORDER BY 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "edge!", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "TextArray" + ] + }, + "nullable": [ + null + ] + }, + "hash": "49d905cae6ba42a3df62bad9385e613381cb4dcf7e851197150a3771ab43c99e" +} diff --git a/backend/.sqlx/query-6890e2be43ff8c653ce62f8cde1b9877190091923a2615d7d94bdf89a12e43c7.json b/backend/.sqlx/query-6890e2be43ff8c653ce62f8cde1b9877190091923a2615d7d94bdf89a12e43c7.json new file mode 100644 index 0000000000..07c923620a --- /dev/null +++ b/backend/.sqlx/query-6890e2be43ff8c653ce62f8cde1b9877190091923a2615d7d94bdf89a12e43c7.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO script (workspace_id, hash, path, summary, description, content, created_by,\n language)\n VALUES ('test-workspace', 1, 'u/test-user/project', '', '', '', 'test-user', 'dbt')", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "6890e2be43ff8c653ce62f8cde1b9877190091923a2615d7d94bdf89a12e43c7" +} diff --git a/backend/.sqlx/query-ae4c0e8aeeef90d08a9b9c919b881b0a03e44beea80da732a4ad7166cf4c06cf.json b/backend/.sqlx/query-ae4c0e8aeeef90d08a9b9c919b881b0a03e44beea80da732a4ad7166cf4c06cf.json new file mode 100644 index 0000000000..2e0e072867 --- /dev/null +++ b/backend/.sqlx/query-ae4c0e8aeeef90d08a9b9c919b881b0a03e44beea80da732a4ad7166cf4c06cf.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT trigger_ref FROM script_trigger WHERE workspace_id = 'test-workspace' AND runnable_path = 'u/test-user/consumer' AND trigger_kind = 'asset'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "trigger_ref", + "type_info": "Text" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false + ] + }, + "hash": "ae4c0e8aeeef90d08a9b9c919b881b0a03e44beea80da732a4ad7166cf4c06cf" +} diff --git a/backend/.sqlx/query-b8161f6481460bed1c985bedc7c638a42f03599ce5eb872cf601f1da2e76b946.json b/backend/.sqlx/query-b8161f6481460bed1c985bedc7c638a42f03599ce5eb872cf601f1da2e76b946.json new file mode 100644 index 0000000000..6b2ec01b23 --- /dev/null +++ b/backend/.sqlx/query-b8161f6481460bed1c985bedc7c638a42f03599ce5eb872cf601f1da2e76b946.json @@ -0,0 +1,63 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT s.path AS \"path!\", s.language AS \"language!: ScriptLang\"\n FROM asset a\n JOIN script s ON s.workspace_id = a.workspace_id AND s.path = a.usage_path\n AND s.archived = false AND s.deleted = false\n WHERE a.workspace_id = $1 AND a.kind = 'dbt' AND a.path = $2\n AND a.usage_kind = 'script' AND a.usage_access_type IN ('w', 'rw')\n AND a.usage_path <> ALL($3)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "language!: ScriptLang", + "type_info": { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java", + "duckdb", + "ruby", + "rlang", + "dbt" + ] + } + } + } + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "TextArray" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "b8161f6481460bed1c985bedc7c638a42f03599ce5eb872cf601f1da2e76b946" +} diff --git a/backend/.sqlx/query-e37b74f77cfa8769aee4d95155a3b3b6856b44d49c0712f08e307f950f210570.json b/backend/.sqlx/query-e37b74f77cfa8769aee4d95155a3b3b6856b44d49c0712f08e307f950f210570.json new file mode 100644 index 0000000000..46a9c8e505 --- /dev/null +++ b/backend/.sqlx/query-e37b74f77cfa8769aee4d95155a3b3b6856b44d49c0712f08e307f950f210570.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind)\n VALUES ('test-workspace', 'main/analytics/orders', 'dbt', 'w', 'u/test-user/project',\n 'script')", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "e37b74f77cfa8769aee4d95155a3b3b6856b44d49c0712f08e307f950f210570" +} diff --git a/backend/parsers/windmill-parser/src/asset_parser.rs b/backend/parsers/windmill-parser/src/asset_parser.rs index 843a83a310..30c6047255 100644 --- a/backend/parsers/windmill-parser/src/asset_parser.rs +++ b/backend/parsers/windmill-parser/src/asset_parser.rs @@ -29,10 +29,10 @@ pub enum AssetKind { Ducklake, DataTable, Volume, - /// A warehouse relation a dbt project builds or reads, - /// `dbt:////`, the warehouse named as the - /// workspace configures it. The scheme names the producer, the path stays - /// the relation — see `windmill_types::AssetKind::Dbt`. + /// A warehouse relation, `dbt:////`, the warehouse + /// named as the workspace configures it. The scheme names the namespace dbt + /// made — a script in any language but dbt's own can declare a write to one — + /// and the path stays the relation. See `windmill_types::AssetKind::Dbt`. Dbt, } @@ -288,11 +288,13 @@ pub struct RetrySpec { } // `// materialize [manual] [append] [key=] [history] [track=]` -// — declares that this script produces a *managed* materialization of `` -// (a `ducklake://` table). By default the runtime generates the write DDL around +// — declares that this script produces ``. A `ducklake://` table is +// materialized *managed* by default: the runtime generates the write DDL around // the script's single trailing `SELECT` and owns idempotency, partition-state // and snapshot capture. `manual` is the escape hatch: the script writes its own -// DDL and the runtime only records state (track-only). The reconciliation +// DDL and the runtime only records state (track-only) — and it is the only mode a +// `dbt://` warehouse relation has, since nothing generates warehouse DDL (deploy +// enforces that; see docs/dbt-runtime.md). The reconciliation // strategy options apply to managed mode: none → DELETE-by-partition + INSERT // (replace); `key=` → MERGE (dedup within slice, SCD type 1); `append` → // INSERT-only. `append` wins if both are given (deploy-time warning). @@ -804,6 +806,18 @@ pub fn canonicalize_table_asset_path(path: &str) -> String { ) } +/// Whether a `dbt://` path names a whole relation, `//`. +/// +/// Every producer spells one that way — the manifest ingest derives it from +/// `relation_name`, a `// materialize` target is checked against it — so anything +/// else can be produced by nothing and read by nothing. Both sides of the deploy +/// ask here rather than counting segments themselves: a subscription and a write +/// that disagreed on the shape would refuse and accept the same string. +pub fn is_full_relation_path(path: &str) -> bool { + let mut segments = path.split('/'); + segments.clone().count() == 3 && !segments.any(str::is_empty) +} + /// A doubled delimiter inside a quoted identifier is that delimiter, literally — /// the same rule the worker's `split_relation` applies to `relation_name`. Both /// have to decode it or one spelling of a table becomes two graph nodes: the dbt @@ -1740,10 +1754,7 @@ mod pipeline_annotation_tests { // just stop being the same node and the cross-boundary cascade never fires. #[test] fn table_paths_from_every_spelling_canonicalize_to_one_key() { - let canonical = Some(( - AssetKind::Dbt, - Cow::Owned("main/analytics/orders".into()), - )); + let canonical = Some((AssetKind::Dbt, Cow::Owned("main/analytics/orders".into()))); for spelling in [ // Hand-written annotation. "dbt://main/analytics/orders", @@ -1765,6 +1776,17 @@ mod pipeline_annotation_tests { } } + /// The shape both halves of the deploy check against: a subscription and a + /// write that disagreed on it would refuse and accept the same string. + #[test] + fn a_whole_relation_is_three_non_empty_segments() { + assert!(is_full_relation_path("main/analytics/orders")); + assert!(is_full_relation_path("main/archive.sales/orders")); + for partial in ["main", "main/analytics", "main/analytics/orders/x", "", "main//orders"] { + assert!(!is_full_relation_path(partial), "{partial} is not a relation"); + } + } + // A relation that overrode its database carries `.` in // one segment, and each half can be quoted independently. Stripping only // the outer pair leaves a key the manifest ingest never produces, so the @@ -1791,10 +1813,7 @@ mod pipeline_annotation_tests { // database qualifier. assert_eq!( parse_asset_syntax("dbt://main/\"sales.v2\"/orders", false), - Some(( - AssetKind::Dbt, - Cow::Owned("main/sales.v2/orders".into()) - )) + Some((AssetKind::Dbt, Cow::Owned("main/sales.v2/orders".into()))) ); } @@ -1813,14 +1832,8 @@ mod pipeline_annotation_tests { "dbt://main/analytics/\"order\"\"s\"", "main/analytics/order\"s", ), - ( - "dbt://main/`da``ta`/`orders`", - "main/da`ta/orders", - ), - ( - "dbt://main/[my]]schema]/[orders]", - "main/my]schema/orders", - ), + ("dbt://main/`da``ta`/`orders`", "main/da`ta/orders"), + ("dbt://main/[my]]schema]/[orders]", "main/my]schema/orders"), // And in one half of a database-qualified segment. ( "dbt://main/\"arch\"\"ive\".\"sales\"/orders", @@ -1839,8 +1852,14 @@ mod pipeline_annotation_tests { // apart. A lone delimiter treated as opening a quote would be dropped — // `sa"les` filed as `sales` — and the two derivations would split. for (decoded, spelled) in [ - ("dbt://main/sa\"les/orders", "dbt://main/\"sa\"\"les\"/orders"), - ("dbt://main/analytics/order\"s", "dbt://main/analytics/\"order\"\"s\""), + ( + "dbt://main/sa\"les/orders", + "dbt://main/\"sa\"\"les\"/orders", + ), + ( + "dbt://main/analytics/order\"s", + "dbt://main/analytics/\"order\"\"s\"", + ), ( "dbt://main/arch\"ive.sales/orders", "dbt://main/\"arch\"\"ive\".\"sales\"/orders", diff --git a/backend/windmill-api-integration-tests/tests/dbt_materialize_target.rs b/backend/windmill-api-integration-tests/tests/dbt_materialize_target.rs new file mode 100644 index 0000000000..60f846ca1f --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/dbt_materialize_target.rs @@ -0,0 +1,250 @@ +use serde_json::json; +use sqlx::{Pool, Postgres}; + +use windmill_test_utils::*; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder { + builder.header("Authorization", "Bearer SECRET_TOKEN") +} + +async fn deploy(port: u16, path: &str, content: &str) -> reqwest::Response { + authed(client().post(format!( + "http://localhost:{port}/api/w/test-workspace/scripts/create" + ))) + .json(&json!({ + "path": path, + "summary": "", + "description": "", + "content": content, + "language": "deno", + "schema": { "type": "object", "properties": {}, "required": [] } + })) + .send() + .await + .unwrap() +} + +/// A `dbt://` relation is one graph node only while every side spells it the same +/// way, and three sides derive that spelling independently: the `// materialize` +/// target becomes an `asset.path`, a `// on` ref becomes a `script_trigger`, and +/// the deploy-time refusal joins the two. The unit tests on `sole_dbt_producer` +/// prove the predicate; only a deploy proves the handler feeds it the key the +/// table actually holds — so a canonicalization that drifted on one side would +/// pass those and split the node here. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_dbt_materialize_target_deploy_contract(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + sqlx::query!( + r#"UPDATE workspace_settings + SET dbt_warehouses = '{"main": {"resource_path": "u/test-user/wh"}}'::jsonb + WHERE workspace_id = 'test-workspace'"# + ) + .execute(&db) + .await?; + + // Nothing generates warehouse DDL, so a managed target is refused rather than + // degraded into the track-only mode it would silently become. + let resp = deploy( + port, + "u/test-user/managed", + "// materialize dbt://main/analytics/orders\nexport async function main() {}", + ) + .await; + assert_eq!(resp.status(), 400); + assert!(resp.text().await?.contains("must be `manual`")); + + // The warehouse segment is the identity a dbt model keys on; a name the + // workspace does not configure strands the write on an unreachable node. + let resp = deploy( + port, + "u/test-user/unknown_wh", + "// materialize manual dbt://nope/analytics/orders\nexport async function main() {}", + ) + .await; + assert_eq!(resp.status(), 400); + assert!(resp.text().await?.contains("does not configure")); + + // Only the DuckDB executor runs `// data_test` probes, and it runs them around + // a managed write — so a declarer in another language would deploy green with + // its assertions silently never executed. + let resp = deploy( + port, + "u/test-user/tested", + "// materialize manual dbt://main/analytics/orders\n// data_test not_null id\nexport async function main() {}", + ) + .await; + assert_eq!(resp.status(), 400); + assert!(resp + .text() + .await? + .contains("`// data_test` is not supported")); + + // Both halves are held to the same relation: every producer is a whole + // `//` under a configured warehouse, so a + // subscription to anything else names something nothing can ever write. + for (path, ref_, expected) in [ + ( + "u/test-user/partial_sub", + "dbt://main/analytics", + "not a whole warehouse relation", + ), + ( + "u/test-user/unknown_wh_sub", + "dbt://nope/analytics/orders", + "does not configure", + ), + ] { + let resp = deploy( + port, + path, + &format!("// on {ref_}\nexport async function main() {{}}"), + ) + .await; + assert_eq!(resp.status(), 400); + assert!(resp.text().await?.contains(expected)); + } + + // Past `asset.path`'s column, where the manifest ingest drops the relation and + // no producer row can exist on either side — computed from the bound so it + // cannot drift under it. + let overlong = format!( + "main/analytics/{}", + "o".repeat(windmill_common::dbt_manifest::MAX_ASSET_PATH_LEN) + ); + let resp = deploy( + port, + "u/test-user/overlong_sub", + &format!("// on dbt://{overlong}\nexport async function main() {{}}"), + ) + .await; + assert_eq!(resp.status(), 400); + assert!(resp.text().await?.contains("characters an asset path holds")); + + // Any language may declare the write — the DuckLake write engine is DuckDB's, + // this declaration is not — and the target is canonicalized on the way into + // `asset`, so a hand-written mixed-case spelling lands on the model's key. + let resp = deploy( + port, + "u/test-user/ingest", + "// materialize manual dbt://main/ANALYTICS/Orders\nexport async function main() {}", + ) + .await; + assert_eq!(resp.status(), 201); + // The create response, not `{:x}` over the stored i64: `ScriptHash` decodes + // hex and demands 8 bytes, while `LowerHex` drops leading zeros, so a hash + // under 2^60 would 422 the rename below instead of reaching the refusal. + let ingest_hash = resp.text().await?; + let write = sqlx::query_scalar!( + "SELECT path FROM asset WHERE workspace_id = 'test-workspace' AND kind = 'dbt' \ + AND usage_path = 'u/test-user/ingest' AND usage_access_type = 'w'" + ) + .fetch_one(&db) + .await?; + assert_eq!(write, "main/analytics/orders"); + + // That producer is native, so subscribing to what it writes is accepted — and + // the `// on` ref has to canonicalize identically, or the row it stores names + // a relation nothing produces. + let resp = deploy( + port, + "u/test-user/consumer", + "// on dbt://main/\"Analytics\"/\"Orders\"\nexport async function main() {}", + ) + .await; + assert_eq!(resp.status(), 201); + let trigger_ref = sqlx::query_scalar!( + "SELECT trigger_ref FROM script_trigger WHERE workspace_id = 'test-workspace' \ + AND runnable_path = 'u/test-user/consumer' AND trigger_kind = 'asset'" + ) + .fetch_one(&db) + .await?; + assert_eq!(trigger_ref, "dbt://main/analytics/orders"); + + // With dbt as the only producer the same subscription can never be woken — a + // dbt run does not dispatch — so the deploy refuses it and names the project. + sqlx::query!( + "INSERT INTO script (workspace_id, hash, path, summary, description, content, created_by, + language) + VALUES ('test-workspace', 1, 'u/test-user/project', '', '', '', 'test-user', 'dbt')" + ) + .execute(&db) + .await?; + sqlx::query!( + "INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind) + VALUES ('test-workspace', 'main/analytics/marts', 'dbt', 'w', 'u/test-user/project', + 'script')" + ) + .execute(&db) + .await?; + let resp = deploy( + port, + "u/test-user/mart_consumer", + "// on dbt://main/analytics/MARTS\nexport async function main() {}", + ) + .await; + assert_eq!(resp.status(), 400); + assert!(resp.text().await?.contains("u/test-user/project")); + + // A rename is the other half of that: the producer's write still sits at the + // OLD path in the committed snapshot this deploy reads, while the same + // transaction removes it — so it must not count as the producer that would + // wake the subscription the rename adds. + sqlx::query!( + "INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind) + VALUES ('test-workspace', 'main/analytics/orders', 'dbt', 'w', 'u/test-user/project', + 'script')" + ) + .execute(&db) + .await?; + let resp = authed(client().post(format!( + "http://localhost:{port}/api/w/test-workspace/scripts/create" + ))) + .json(&json!({ + "path": "u/test-user/ingest_renamed", + "parent_hash": ingest_hash, + "summary": "", + "description": "", + "content": "// on dbt://main/analytics/orders\nexport async function main() {}", + "language": "deno", + "schema": { "type": "object", "properties": {}, "required": [] } + })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 400); + assert!(resp.text().await?.contains("u/test-user/project")); + + // Neither annotation is accepted on a dbt script: the graph ingest + // republishes that path's asset and trigger rows wholesale, so either would + // deploy something the dependency job then silently removes. + for content in [ + "# materialize manual dbt://main/analytics/orders\nprofile:\n warehouse: main\n", + "# on dbt://main/analytics/orders\nprofile:\n warehouse: main\n", + ] { + let resp = authed(client().post(format!( + "http://localhost:{port}/api/w/test-workspace/scripts/create" + ))) + .json(&json!({ + "path": "u/test-user/dbt_project", + "summary": "", + "description": "", + "content": content, + "language": "dbt", + "modules": { "dbt_project.yml": { "content": "name: p\n", "language": "dbt" } }, + "schema": { "type": "object", "properties": {}, "required": [] } + })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 400); + assert!(resp.text().await?.contains("a dbt script cannot")); + } + + Ok(()) +} diff --git a/backend/windmill-api-scripts/src/scripts.rs b/backend/windmill-api-scripts/src/scripts.rs index ccc5815aa7..d24b3c3a5c 100644 --- a/backend/windmill-api-scripts/src/scripts.rs +++ b/backend/windmill-api-scripts/src/scripts.rs @@ -39,8 +39,8 @@ use sqlx::{FromRow, Postgres, Transaction}; use std::{collections::HashMap, sync::Arc}; use windmill_audit::audit_oss::{audit_log, AuditAuthorable}; use windmill_audit::ActionKind; -use windmill_dep_map::{lock_hash::record_lock_hashes, process_relative_imports}; use windmill_dep_map::scoped_dependency_map::ScopedDependencyMap; +use windmill_dep_map::{lock_hash::record_lock_hashes, process_relative_imports}; use windmill_common::{ assets::{ @@ -1081,6 +1081,57 @@ fn lock_hash_entry(path: &str, lock: &str) -> [(String, i64); 1] { [(path.to_string(), hash_script(lock))] } +/// The `dbt://` relation both halves of a deploy have to agree on: a whole +/// `//`, under a warehouse this workspace configures. +/// +/// Every producer is held to exactly this — a `// materialize` target here, a +/// descriptor's `profile.warehouse` in the worker — so a subscription to anything +/// else names a relation nothing can ever write. No later deploy fixes that and +/// no dormant-edge warning reports it, since the warning fires on a dbt project's +/// ingest and no project can claim a relation under a warehouse that isn't there. +/// Asking here rather than at each site is what keeps the two from drifting into +/// refusing and accepting the same string. +async fn validate_dbt_relation( + db: &sqlx::Pool, + w_id: &str, + relation: &str, + what: &str, +) -> Result<()> { + if !windmill_parser::asset_parser::is_full_relation_path(relation) { + return Err(Error::BadRequest(format!( + "{what} `dbt://{relation}` is not a whole warehouse relation \ + (`dbt:////`)." + ))); + } + // `asset.path` is VARCHAR(255) and the manifest ingest drops a relation that + // outgrows it rather than failing the whole graph, so past the column no + // producer row can exist on either side — a write would be rejected by + // Postgres mid-deploy, and `script_trigger.trigger_ref` is unbounded text + // that would take the subscription and keep it dormant for good. + let max = windmill_common::dbt_manifest::MAX_ASSET_PATH_LEN; + if relation.chars().count() > max { + return Err(Error::BadRequest(format!( + "{what} `dbt://{relation}` is longer than the {max} characters an asset path \ + holds, so it cannot be recorded." + ))); + } + let warehouse = relation.split('/').next().unwrap_or_default(); + // Only the resolver's own "no such warehouse" is the annotation's fault. Its + // other failures — the query, and a setting entry with no `resource_path` — + // keep their own error: blaming the warehouse name for those misdescribes + // them, and flattening the malformed-setting one to a 400 hides a server + // fault behind a client one. + windmill_common::workspaces::dbt_warehouse_exists(db, w_id, warehouse) + .await + .map_err(|e| match e { + Error::NotFound(_) => Error::BadRequest(format!( + "{what} `dbt://{relation}` names a warehouse this workspace does not \ + configure: {e}" + )), + other => other, + }) +} + async fn create_script_internal<'c>( mut ns: NewScript, w_id: String, @@ -1565,34 +1616,90 @@ async fn create_script_internal<'c>( // membership; parsed writes tell us what is produced (we don't record // them in auto_kind itself). let pipeline_annotations = parse_pipeline_annotations(&ns.content); - // `// materialize` materializes a `ducklake:///` target from a - // DuckDB script. These two constraints hold for *both* modes: a non-DuckLake - // target would otherwise deploy, register a producer in the asset graph, then - // silently no-op at run time (`build_materialized_query` returns `Ok(None)`), - // and a non-DuckDB script never reaches the executor that records state. The - // managed-only checks (single trailing SELECT, no SQL args) come after — a - // `manual` script owns its DDL and skips them. + // `// materialize` names what this script produces. Two target kinds, and the + // runtime behind each is what constrains the annotation: + // • `ducklake:///
` — the DuckDB executor generates the write + // (or, in `manual` mode, records the state the script wrote itself), so + // the script has to be a DuckDB one and the target has to name a table. + // A non-DuckDB script never reaches that executor. + // • `dbt:////` — a warehouse relation. Nothing + // generates warehouse DDL, so the declaration is track-only (`manual`) + // and any language but dbt's own may make it: the script writes the + // relation, the worker records the materialization, and the relation's + // asset node is shared with whatever dbt model reads it. A dbt project's + // own writes are read from its manifest, so it may not declare one. + // Any other kind would deploy, register a producer in the asset graph, then + // silently no-op at run time (`build_materialized_query` returns `Ok(None)`). + // The managed-only checks (single trailing SELECT, no SQL args) come after — + // a `manual` script owns its DDL and skips them. if let Some(m) = pipeline_annotations.materialize.as_ref() { - if ns.language != ScriptLang::DuckDb { - return Err(Error::BadRequest(format!( - "`// materialize` is only supported for DuckDB scripts, not {}. Use the \ - wmll.ducklake helpers to materialize from other languages.", - ns.language.as_str() - ))); - } - if m.target_kind != windmill_parser::asset_parser::AssetKind::Ducklake { + use windmill_parser::asset_parser::AssetKind as PAssetKind; + // The producer half of the rule the trigger loop below applies to `// on`: + // a dbt project's writes come from its manifest, and the graph ingest + // republishes this path's asset rows wholesale, so a declared one would be + // wiped by the very deploy that accepted it while its runs kept stamping + // the relation. + if ns.language == ScriptLang::Dbt { return Err(Error::BadRequest( - "`// materialize` only supports a DuckLake target \ - (`ducklake:///
`); other asset kinds aren't materializable." + "a dbt script cannot declare `// materialize`: what a project builds is read \ + from its manifest and published by the graph ingest, not annotated." .to_string(), )); } - if !m.target_path.contains('/') { - return Err(Error::BadRequest(format!( - "`// materialize` needs a table in the target: \ - `ducklake://{0}/
` (got `ducklake://{0}`).", - m.target_path - ))); + match m.target_kind { + PAssetKind::Ducklake => { + if ns.language != ScriptLang::DuckDb { + return Err(Error::BadRequest(format!( + "`// materialize` is only supported for DuckDB scripts, not {}. Use the \ + wmll.ducklake helpers to materialize from other languages, or declare a \ + warehouse relation with `// materialize manual dbt://…`.", + ns.language.as_str() + ))); + } + if !m.target_path.contains('/') { + return Err(Error::BadRequest(format!( + "`// materialize` needs a table in the target: \ + `ducklake://{0}/
` (got `ducklake://{0}`).", + m.target_path + ))); + } + } + PAssetKind::Dbt => { + // `// data_test` runs as verifier probes the DuckDB executor + // splices around a MANAGED write. Nothing generates a warehouse + // write, so nothing would run them — and unlike the DuckLake + // `manual` case, which at least fails loudly in that executor, a + // declarer in another language would deploy green with its + // data-quality assertions silently never executed. + if !pipeline_annotations.data_tests.is_empty() { + return Err(Error::BadRequest( + "`// data_test` is not supported with a `dbt://` target: the checks run \ + against a managed materialization, and a warehouse relation is \ + written by the script itself. Assert on the relation with a dbt \ + test in the project that reads it." + .to_string(), + )); + } + if !m.manual { + return Err(Error::BadRequest( + "`// materialize dbt://…` must be `manual`: Windmill generates no \ + warehouse DDL, so the script issues its own write and only the outcome \ + is recorded. Write \ + `// materialize manual dbt:////`." + .to_string(), + )); + } + validate_dbt_relation(&db, &w_id, &m.target_path, "`// materialize` target") + .await?; + } + _ => { + return Err(Error::BadRequest( + "`// materialize` only supports a DuckLake (`ducklake:///
`) or \ + warehouse-relation (`dbt:////`) target; other asset \ + kinds aren't materializable." + .to_string(), + )); + } } if !m.manual { if let Err(e) = windmill_parser::sql_materialize::classify_wrap(&ns.content) { @@ -2375,16 +2482,43 @@ async fn create_script_internal<'c>( let Some((trigger_kind, trigger_ref)) = trigger_spec_to_row(spec) else { continue; }; - // A `dbt://` subscription can never fire: dbt is the only producer of a - // warehouse relation (`// materialize` takes DuckLake targets only) and a - // dbt run does not dispatch. Refusing beats persisting a row that draws a - // cascade arrow on the canvas and then never wakes anything. - if trigger_ref.starts_with("dbt://") { - return Err(Error::BadRequest(format!( - "`{trigger_ref}` cannot be subscribed to: a dbt run does not trigger downstream \ - runs, and nothing else writes a warehouse relation. Declare the read without \ - `on` to keep the lineage edge, or schedule this script." - ))); + // A `dbt://` subscription fires only when a NON-dbt job materialized the + // relation: `// materialize manual dbt://…` declares such a write, while a + // dbt run records its models and does not dispatch. So refuse exactly the + // edge that cannot fire — one whose relation is already claimed by dbt and + // by nothing else — rather than every `dbt://` edge (`sole_dbt_producer`, + // which takes the workspace pool: under RLS an unreadable native producer + // would refuse a live subscription). + if let Some(relation) = trigger_ref.strip_prefix("dbt://") { + // The subscriber side of the same rule: a dbt project is not woken by + // the asset cascade. Its graph ingest clears these rows for its own + // path, so accepting one here would deploy an edge the dependency job + // then silently removes. + if ns.language == ScriptLang::Dbt { + return Err(Error::BadRequest(format!( + "a dbt script cannot subscribe to `{trigger_ref}`: dbt orders its own DAG \ + and a project is run on its schedule, not woken by an asset cascade." + ))); + } + validate_dbt_relation(&db, &w_id, relation, "subscription target").await?; + // Both paths under a rename: the old one's committed write row is + // still there and this transaction is about to remove it. + let deploying_paths = match p_path_opt.as_deref().filter(|old| *old != ns.path) { + Some(old) => vec![ns.path.clone(), old.to_string()], + None => vec![ns.path.clone()], + }; + if let Some(dbt_owner) = + windmill_common::assets::sole_dbt_producer(&db, &w_id, relation, &deploying_paths) + .await? + { + return Err(Error::BadRequest(format!( + "`{trigger_ref}` cannot be subscribed to: it is built by the dbt project at \ + `{dbt_owner}`, and a dbt run does not trigger downstream runs. Declare the \ + read without `on` to keep the lineage edge, or schedule this script. A \ + relation written by a `// materialize manual {trigger_ref}` script can be \ + subscribed to." + ))); + } } // Effective debounce for this edge: per-`// on debounce=` wins, // else the script-level `// debounce` default. Debounce only diff --git a/backend/windmill-common/src/assets.rs b/backend/windmill-common/src/assets.rs index 117b41632a..4d5523524c 100644 --- a/backend/windmill-common/src/assets.rs +++ b/backend/windmill-common/src/assets.rs @@ -175,10 +175,12 @@ fn is_write_access(access: Option) -> bool { /// producers). Resource / datatable / volume reads stay explicit-`// on`: /// a config/lookup read cascading is more often surprising than wanted. fn is_auto_trigger_kind(kind: AssetKind) -> bool { - // `Dbt` is deliberately NOT here. dbt is the only thing that can produce a - // warehouse relation (`// materialize` takes DuckLake targets only) and a dbt - // run does not dispatch, so a derived `dbt://` edge could never fire — it - // would draw a cascade arrow into a script nothing can wake. + // `Dbt` is deliberately NOT here. A warehouse relation is usually built by + // the dbt project that reads it, and a dbt run does not dispatch, so deriving + // an edge from every `dbt://` read would draw cascade arrows that mostly never + // fire. The relations a native `// materialize manual dbt://…` script writes + // do wake subscribers, but only through an explicit `// on`, which is where + // the author states that this particular relation has such a producer. matches!(kind, AssetKind::Ducklake | AssetKind::S3Object) } @@ -235,6 +237,134 @@ pub fn derive_pipeline_asset_trigger_refs( out } +/// A dbt script that builds the `dbt://` relation at `asset_path`, when dbt is +/// its ONLY producer. +/// +/// That is the one shape in which subscribing to a warehouse relation can never +/// be woken: a dbt run records the models it built and does not dispatch +/// (`asset_dispatch` returns early for `ScriptLang::Dbt`), while a script that +/// declares `// materialize manual dbt://…` fans out on the ordinary path. +/// +/// `None` covers both "some non-dbt script materializes it" and "nothing +/// produces it yet" — the second is the ordinary deploy-order case, identical to +/// every other asset kind, not a dormant edge. +/// +/// **Give it the workspace pool, not an RLS-scoped transaction.** `script` +/// carries RLS while `asset` does not, so a scoped executor hides producers, and +/// the hidden ones fail in the harmful direction: a native producer the deployer +/// cannot read leaves a dbt-only set behind and refuses a subscription that would +/// have fired. What it discloses in exchange is the path of a dbt script building +/// a relation the caller already named, which the workspace asset graph hands out +/// for every `dbt://` node anyway (the source that script wrote stays gated). +/// Callers must therefore already be scoped to `workspace_id`. +/// +/// `deploying_paths` is excluded from the producer set, and has to be: reading +/// committed rows means the deploying script's own are the version being +/// replaced, so one that just dropped its `// materialize` would still count as a +/// producer and let a now-dormant subscription through. Pass every path this +/// deploy is rewriting — under a rename that is the old path as well as the new +/// one, whose committed write row the transaction is about to remove. Excluding +/// them is free of the opposite error, because a script never wakes its own +/// subscription — the dispatcher skips that as a self-loop. +/// +/// A producer another deploy is committing concurrently is invisible either way, +/// and the outcome depends on which side it is. An uncommitted NATIVE producer +/// leaves a dbt-only set and refuses, with a message the user can retry past. An +/// uncommitted DBT one leaves an empty set and accepts — and if that ingest then +/// commits and runs [`dormant_dbt_subscriptions`] before this deploy's trigger +/// row lands, neither side reports the edge it left dormant. Serializing the two +/// is not worth it: they would have to share a per-relation lock, and the ingest +/// takes `script … FOR UPDATE` before its own advisory lock, so a deploy holding +/// relation locks first inverts that order into a deadlock across the two +/// subsystems. The next deploy of that project warns (docs/dbt-runtime.md). +pub async fn sole_dbt_producer<'e>( + executor: impl PgExecutor<'e>, + workspace_id: &str, + asset_path: &str, + deploying_paths: &[String], +) -> error::Result> { + use crate::scripts::ScriptLang; + let producers = sqlx::query!( + r#"SELECT s.path AS "path!", s.language AS "language!: ScriptLang" + FROM asset a + JOIN script s ON s.workspace_id = a.workspace_id AND s.path = a.usage_path + AND s.archived = false AND s.deleted = false + WHERE a.workspace_id = $1 AND a.kind = 'dbt' AND a.path = $2 + AND a.usage_kind = 'script' AND a.usage_access_type IN ('w', 'rw') + AND a.usage_path <> ALL($3)"#, + workspace_id, + asset_path, + deploying_paths + ) + .fetch_all(executor) + .await?; + if producers + .iter() + .any(|p| !matches!(p.language, ScriptLang::Dbt)) + { + return Ok(None); + } + Ok(producers.into_iter().next().map(|p| p.path)) +} + +/// The set form of [`sole_dbt_producer`], for asking about many relations at +/// once: every `// on dbt://` edge among `relations` whose producers +/// are all dbt scripts, rendered as `dbt:// → `. +/// +/// A dbt deploy asks this about the relations it just ingested, because that +/// ingest is what can retroactively leave a subscription accepted earlier — when +/// nothing produced the relation — with dbt as its only producer. +/// +/// Spells the predicate the same way its singular sibling does, per subscriber: +/// the producer set excludes the subscriber's own path (a script never wakes +/// itself) and has to be non-empty (nothing produces it yet is deploy order, not +/// a dormant edge). Two "is dbt the sole producer" rules that drifted apart would +/// silence this warning with nothing failing. +/// +/// Same disclosure and executor contract as [`sole_dbt_producer`]: workspace +/// pool, caller already scoped to `workspace_id`. +pub async fn dormant_dbt_subscriptions<'e>( + executor: impl PgExecutor<'e>, + workspace_id: &str, + relations: &[String], +) -> error::Result> { + if relations.is_empty() { + return Ok(vec![]); + } + let refs = relations + .iter() + .map(|r| format!("dbt://{r}")) + .collect::>(); + Ok(sqlx::query_scalar!( + r#"WITH producer AS ( + SELECT 'dbt://' || a.path AS trigger_ref, a.usage_path, s.language + FROM asset a + JOIN script s ON s.workspace_id = a.workspace_id AND s.path = a.usage_path + AND s.archived = false AND s.deleted = false + WHERE a.workspace_id = $1 AND a.kind = 'dbt' + AND a.usage_kind = 'script' AND a.usage_access_type IN ('w', 'rw') + AND 'dbt://' || a.path = ANY($2) + ) + SELECT DISTINCT st.trigger_ref || ' → ' || st.runnable_path AS "edge!" + FROM script_trigger st + WHERE st.workspace_id = $1 AND st.trigger_kind = 'asset' + AND st.trigger_ref = ANY($2) + AND EXISTS (SELECT 1 FROM producer p + WHERE p.trigger_ref = st.trigger_ref + AND p.usage_path <> st.runnable_path + AND p.language = 'dbt') + AND NOT EXISTS (SELECT 1 FROM producer p + WHERE p.trigger_ref = st.trigger_ref + AND p.usage_path <> st.runnable_path + AND p.language <> 'dbt') + ORDER BY 1"#, + workspace_id, + &refs + ) + .fetch_all(executor) + .await?) +} + /// Clear and reinsert the full static-asset usage set of a script in one tx, /// invalidating the producer-writes cache at most once and only on a real /// change. The cache (asset_dispatch::ASSET_PRODUCER_WRITES_CACHE) keys a diff --git a/backend/windmill-common/src/dbt_manifest.rs b/backend/windmill-common/src/dbt_manifest.rs index 8fb124a33f..e85dfc1910 100644 --- a/backend/windmill-common/src/dbt_manifest.rs +++ b/backend/windmill-common/src/dbt_manifest.rs @@ -8,9 +8,11 @@ //! Two things this module is deliberate about: //! //! * **Asset identity is the physical relation.** A model becomes -//! `dbt:////`: the scheme names dbt, which is the -//! only thing that creates one, but the PATH is the relation and never dbt's -//! own `unique_id`. Two projects meet at a handoff — one materializes a mart, +//! `dbt:////`: the scheme names the namespace dbt +//! made — it is the only thing that DERIVES one, while any other language can +//! declare a write to one (decision 25) — but the PATH is the relation and +//! never dbt's own `unique_id`. Two projects meet at a handoff — one +//! materializes a mart, //! the next declares it a `source` — where `model.a.orders` and //! `source.b.analytics.orders` differ but the relation does not, so keying on //! the node id would leave each project an island; a native script reading the diff --git a/backend/windmill-common/tests/dbt_producer_rules.rs b/backend/windmill-common/tests/dbt_producer_rules.rs new file mode 100644 index 0000000000..f63e0879b5 --- /dev/null +++ b/backend/windmill-common/tests/dbt_producer_rules.rs @@ -0,0 +1,196 @@ +/*! + * One rule, two spellings: "is dbt the sole producer of this warehouse + * relation". `sole_dbt_producer` decides whether a `// on dbt://…` subscription + * is refused at deploy; `dormant_dbt_subscriptions` names the edges a dbt deploy + * retroactively leaves unwakeable. Both must answer "yes, dormant" only when + * every script writing the relation is a dbt one — a dbt run does not dispatch — + * and "no" both when a native `// materialize manual dbt://…` producer exists and + * when nothing produces the relation yet, which is the ordinary deploy-order + * case. Every way of getting this wrong is silent: a dormant edge on the canvas, + * a refused deploy of a valid pipeline, or a warning that stops appearing. + */ + +use sqlx::{Pool, Postgres}; +use windmill_common::assets::{dormant_dbt_subscriptions, sole_dbt_producer}; + +const WS: &str = "test-workspace"; +const RELATION: &str = "main/analytics/orders"; +const SUBSCRIBER: &str = "u/test-user/consumer"; + +async fn plant_producer(db: &Pool, path: &str, language: &str, hash: i64) { + sqlx::query( + "INSERT INTO script (workspace_id, hash, path, summary, description, content, created_by, + language) + VALUES ($1, $2, $3, '', '', '', 'test-user', $4::text::script_lang)", + ) + .bind(WS) + .bind(hash) + .bind(path) + .bind(language) + .execute(db) + .await + .expect("insert script"); + sqlx::query( + "INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind) + VALUES ($1, $2, 'dbt', 'w', $3, 'script') ON CONFLICT DO NOTHING", + ) + .bind(WS) + .bind(RELATION) + .bind(path) + .execute(db) + .await + .expect("insert asset"); +} + +async fn plant_subscriber(db: &Pool, path: &str) { + sqlx::query( + "INSERT INTO script_trigger (workspace_id, runnable_kind, runnable_path, trigger_kind, + trigger_ref) + VALUES ($1, 'script', $2, 'asset', 'dbt://' || $3)", + ) + .bind(WS) + .bind(path) + .bind(RELATION) + .execute(db) + .await + .expect("insert script_trigger"); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn no_producer_is_not_dormant(db: Pool) { + assert_eq!( + sole_dbt_producer(&db, WS, RELATION, &[SUBSCRIBER.to_string()]) + .await + .unwrap(), + None, + "a relation nothing produces yet must not refuse the subscription" + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn dbt_only_producer_is_dormant(db: Pool) { + plant_producer(&db, "u/test-user/project", "dbt", 1).await; + assert_eq!( + sole_dbt_producer(&db, WS, RELATION, &[SUBSCRIBER.to_string()]) + .await + .unwrap(), + Some("u/test-user/project".to_string()) + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn a_native_producer_beside_dbt_is_not_dormant(db: Pool) { + plant_producer(&db, "u/test-user/project", "dbt", 1).await; + plant_producer(&db, "u/test-user/ingest", "postgresql", 2).await; + assert_eq!( + sole_dbt_producer(&db, WS, RELATION, &[SUBSCRIBER.to_string()]) + .await + .unwrap(), + None + ); +} + +/// `asset` is keyed by path while `script` holds every version of it, so the +/// language has to be read off the live one: a path converted to dbt still has +/// its old native versions sitting in `script`. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn a_superseded_native_version_does_not_count(db: Pool) { + plant_producer(&db, "u/test-user/project", "postgresql", 1).await; + sqlx::query("UPDATE script SET archived = true WHERE hash = 1") + .execute(&db) + .await + .expect("archive the old version"); + plant_producer(&db, "u/test-user/project", "dbt", 2).await; + assert_eq!( + sole_dbt_producer(&db, WS, RELATION, &[SUBSCRIBER.to_string()]) + .await + .unwrap(), + Some("u/test-user/project".to_string()) + ); +} + +/// The rows of the script being deployed describe the version it replaces, so a +/// script dropping its `// materialize` while adding a subscription would +/// otherwise count itself as the producer that wakes it — and commit a dormant +/// edge. It can never be that producer anyway: the dispatcher skips self-loops. +/// Under a rename that write sits at the OLD path, which the deploy is removing +/// in the same uncommitted transaction, so both paths have to be excluded. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn the_subscriber_is_never_its_own_producer(db: Pool) { + plant_producer(&db, "u/test-user/project", "dbt", 1).await; + plant_producer(&db, SUBSCRIBER, "postgresql", 2).await; + assert_eq!( + sole_dbt_producer(&db, WS, RELATION, &[SUBSCRIBER.to_string()]) + .await + .unwrap(), + Some("u/test-user/project".to_string()) + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn a_renamed_producer_is_excluded_too(db: Pool) { + plant_producer(&db, "u/test-user/project", "dbt", 1).await; + plant_producer(&db, "u/test-user/old_ingest", "postgresql", 2).await; + assert_eq!( + sole_dbt_producer( + &db, + WS, + RELATION, + &[SUBSCRIBER.to_string(), "u/test-user/old_ingest".to_string()] + ) + .await + .unwrap(), + Some("u/test-user/project".to_string()), + "the write this deploy is moving off the old path cannot wake the subscription" + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn the_set_form_agrees_with_the_singular_one(db: Pool) { + let relations = vec![RELATION.to_string()]; + plant_subscriber(&db, "u/test-user/consumer").await; + plant_producer(&db, "u/test-user/project", "dbt", 1).await; + assert_eq!( + dormant_dbt_subscriptions(&db, WS, &relations) + .await + .unwrap(), + vec![format!("dbt://{RELATION} → u/test-user/consumer")], + "dbt alone builds it, so the subscription can never be woken" + ); + + plant_producer(&db, "u/test-user/ingest", "postgresql", 2).await; + assert!( + dormant_dbt_subscriptions(&db, WS, &relations) + .await + .unwrap() + .is_empty(), + "a native producer wakes it, so the edge is live" + ); +} + +/// The two ways the set form could stop meaning what the singular one means: a +/// relation nothing produces is deploy order rather than a dormant edge, and a +/// subscriber's own write is not a producer that can wake it — the dispatcher +/// skips self-loops, so that edge is dormant and has to be named. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn the_set_form_matches_on_the_edge_cases_too(db: Pool) { + let relations = vec![RELATION.to_string()]; + plant_subscriber(&db, SUBSCRIBER).await; + assert!( + dormant_dbt_subscriptions(&db, WS, &relations) + .await + .unwrap() + .is_empty(), + "nothing produces it yet, so nothing is dormant" + ); + + plant_producer(&db, "u/test-user/project", "dbt", 1).await; + plant_producer(&db, SUBSCRIBER, "postgresql", 2).await; + assert_eq!( + dormant_dbt_subscriptions(&db, WS, &relations) + .await + .unwrap(), + vec![format!("dbt://{RELATION} → {SUBSCRIBER}")], + "the subscriber's own write cannot wake it, so dbt is still the sole producer" + ); +} diff --git a/backend/windmill-queue/src/asset_dispatch.rs b/backend/windmill-queue/src/asset_dispatch.rs index af9ca957b9..a3f7154cda 100644 --- a/backend/windmill-queue/src/asset_dispatch.rs +++ b/backend/windmill-queue/src/asset_dispatch.rs @@ -248,11 +248,13 @@ async fn try_dispatch(db: &DB, job: &MiniCompletedJob) -> Result // A dbt run records the relations it builds, so it looks like a producer // here — but dbt does not trigger downstream runs. Its own DAG is dbt's to // order; the only thing a cascade would add is waking Windmill scripts that - // read a mart, and nothing outside dbt can declare a `dbt://` write, so - // that edge exists in one direction only. Cascading from a project whose - // per-run selection can build any subset of itself needs a per-run write set - // to be correct, which is a design worth doing deliberately rather than - // inferring. Until then dbt materializes and reports; it does not dispatch. + // read a mart. Cascading from a project whose per-run selection can build any + // subset of itself needs a per-run write set to be correct, which is a design + // worth doing deliberately rather than inferring: the deploy-time write set is + // not what ran, and the per-relation state table keeps one row per relation. + // Until then dbt materializes and reports; it does not dispatch. The opposite + // direction does: a native `// materialize manual dbt://…` script reaches the + // fan-out below on the ordinary path, on the strength of its own asset rows. if job.script_lang == Some(ScriptLang::Dbt) { return Ok(DispatchResult::default()); } diff --git a/backend/windmill-types/src/assets.rs b/backend/windmill-types/src/assets.rs index 463a365b06..166406b632 100644 --- a/backend/windmill-types/src/assets.rs +++ b/backend/windmill-types/src/assets.rs @@ -14,17 +14,20 @@ pub enum AssetKind { Ducklake, DataTable, Volume, - /// A warehouse relation a dbt project builds or reads, - /// `dbt:////`, where `` is the name the - /// workspace configures it under. + /// A warehouse relation, `dbt:////`, where + /// `` is the name the workspace configures it under. /// - /// The SCHEME names the producer — dbt is the only thing that creates one — - /// while the PATH stays the physical relation, because that is what two - /// projects agree on: a mart one builds is a `source` the next reads, and - /// their dbt `unique_id`s differ (`model.a.orders` vs - /// `source.b.analytics.orders`) where the relation does not - /// (docs/dbt-runtime.md, decision 11). A dbt run does not trigger that - /// reader — the shared node is lineage, not a cascade edge. + /// The SCHEME names the namespace dbt made rather than an exclusive + /// producer: dbt is what derives these relations from a project, and a script + /// in any language but dbt's own can DECLARE one it writes + /// (`// materialize manual dbt://…`) — a project's writes are read from its + /// manifest, never annotated. The PATH stays the physical relation, + /// because that is what two producers agree on: a mart one builds is a + /// `source` the next reads, and their dbt `unique_id`s differ + /// (`model.a.orders` vs `source.b.analytics.orders`) where the relation does + /// not (docs/dbt-runtime.md, decision 11). A dbt run does not trigger the + /// readers of what it built — that shared node is lineage, not a cascade + /// edge — while a declared write does (decision 25). Dbt, } diff --git a/backend/windmill-worker/src/dbt_executor.rs b/backend/windmill-worker/src/dbt_executor.rs index 33ddd6c32e..f302df6a1a 100644 --- a/backend/windmill-worker/src/dbt_executor.rs +++ b/backend/windmill-worker/src/dbt_executor.rs @@ -682,6 +682,7 @@ pub(crate) async fn dbt_dep( &conn, ) .await; + warn_dormant_subscribers(db, w_id, job_id, &ingested, &conn).await; } !published } else { @@ -3177,7 +3178,7 @@ async fn ingest_from_run( let snapshot_job = p.graph_refresh.snapshot_job(job.id); match conn { Connection::Sql(db) => { - persist_ingest( + let published = persist_ingest( db, &job.workspace_id, script_path, @@ -3190,6 +3191,13 @@ async fn ingest_from_run( p.graph_refresh.publishes_ownership(), ) .await?; + // Publishing ownership from a RUN makes this project the owner of + // those relations exactly as a deploy does, so it can be what leaves + // a subscription accepted while the relation had no producer with dbt + // as its only one. Same warning the deploy emits. + if published && p.graph_refresh.publishes_ownership() { + warn_dormant_subscribers(db, &job.workspace_id, &job.id, &ingested, conn).await; + } } // An agent worker reaches these tables only through the API. Publishing // is the whole of what it needs: a worker that can replace the graph @@ -3242,7 +3250,7 @@ enum GraphPublisher { /// Replace this script's graph, unless a newer version of it has been deployed. /// /// Write one ingest: the sidecar rows and the `asset` usages the manifest -/// implies. No subscriptions — a `dbt://` one could never fire. +/// implies. No subscriptions — a dbt project is not woken by the cascade. /// /// Returns whether this job was still the one entitled to the path-keyed half — /// false once a newer version has superseded it, or once the version is gone. @@ -3333,9 +3341,10 @@ async fn persist_ingest( &ingested.assets, ) .await?; - // A `dbt://` subscription can never fire, so none are derived from the - // manifest. The delete stays to clear what earlier versions wrote, which would - // otherwise keep drawing cascade arrows that wake nothing. + // A dbt project is not woken by the asset cascade (refused at deploy), so + // none are derived from the manifest either. The delete stays to clear what + // earlier versions wrote, which would otherwise keep drawing cascade arrows + // that wake nothing. sqlx::query!( "DELETE FROM script_trigger WHERE workspace_id = $1 AND runnable_kind = 'script' AND runnable_path = $2 @@ -3349,6 +3358,56 @@ async fn persist_ingest( Ok(true) } +/// Log the `// on dbt://…` subscriptions this project's relations leave dormant. +/// +/// Subscribing to a relation dbt already owns is refused at the subscriber's +/// deploy, but one deployed while nothing produced that relation is accepted — +/// as it is for every other asset kind — and an ingest is what can afterwards +/// make dbt its only producer. A dbt run does not dispatch, so such an edge is +/// drawn on the canvas and never fires; a job's own log is where that ordering is +/// visible. +/// +/// Called from both points that publish ownership, the deploy and a run whose +/// static descriptor found its profile moved — either can be the one that claims +/// the relation. +async fn warn_dormant_subscribers( + db: &sqlx::Pool, + w_id: &str, + job_id: &Uuid, + ingested: &windmill_common::dbt_manifest::IngestedManifest, + conn: &Connection, +) { + use windmill_common::assets::AssetUsageAccessType; + let relations: Vec = ingested + .assets + .iter() + .filter(|a| { + matches!( + a.access_type.or(a.alt_access_type), + Some(AssetUsageAccessType::W) | Some(AssetUsageAccessType::RW) + ) + }) + .map(|a| a.path.clone()) + .collect(); + match windmill_common::assets::dormant_dbt_subscriptions(db, w_id, &relations).await { + Ok(edges) if !edges.is_empty() => { + append_logs( + job_id, + w_id, + format!( + "\nThese subscriptions will not fire — a dbt run does not trigger downstream \ + runs, and nothing else writes their relation:\n {}\n", + edges.join("\n ") + ), + conn, + ) + .await; + } + Ok(_) => {} + Err(e) => tracing::warn!("listing dormant `dbt://` subscribers failed: {e:#}"), + } +} + /// Serialize publishers for one script path and confirm this job's version is /// still the newest. Both happen inside the caller's transaction, so a newer /// publisher either commits before this check sees it, or waits behind it and diff --git a/backend/windmill-worker/src/duckdb_executor.rs b/backend/windmill-worker/src/duckdb_executor.rs index 54457f81b0..b2e69d26d6 100644 --- a/backend/windmill-worker/src/duckdb_executor.rs +++ b/backend/windmill-worker/src/duckdb_executor.rs @@ -112,7 +112,9 @@ async fn fetch_custom_test_body(conn: &Connection, w_id: &str, path: &str) -> Re // the user's own ATTACH. `// data_test` lines append verifier probes that run // against the freshly-materialized target and raise (failing the run) on // violation. Returns `None` when there is no materialize annotation or the -// target isn't a ducklake (only ducklake is materialized in v1). +// target isn't a ducklake: only ducklake has a write engine, and a `dbt://` +// target is recorded by the generic job path instead (worker.rs, +// `record_declared_warehouse_write`). fn build_materialized_query( query: &str, partition_value: Option<&str>, diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 7e620439b0..8dc52ef936 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -5527,7 +5527,7 @@ async fn handle_code_execution_job( .await?; let language = language.clone(); - run_language_executor( + let result = run_language_executor( job, conn, client, @@ -5553,7 +5553,110 @@ async fn handle_code_execution_job( false, in_pipeline, ) - .await + .await; + record_declared_warehouse_write(job, conn, code, &result).await; + result +} + +/// Record the outcome of a `// materialize manual dbt:////` +/// declaration, the way the DuckDB executor records a DuckLake target. +/// +/// Nothing generates warehouse DDL, so the script issues its own write and this +/// is the only thing that turns it into a `materialized_partition` row — the +/// relation's last writer on the run page and the graph. Language-agnostic on +/// purpose — the DuckLake write engine is DuckDB's, this declaration is anyone's +/// — except dbt's own, which is refused at deploy. +/// +/// Best-effort, and it can be: the cascade fans out from the deploy-time `asset` +/// rows, not from this one, so a lost row costs the relation its last writer and +/// nothing else. It must not fail a job whose write already landed. +/// +/// Shares the reach of every other runtime pipeline annotation, which is this +/// function's caller: a job handed to a dedicated worker or a flow runner never +/// passes through it, so — exactly as `// partitioned` is not resolved there — +/// such a run performs its write and records no row. +async fn record_declared_warehouse_write( + job: &MiniPulledJob, + conn: &Connection, + code: &str, + result: &error::Result>, +) { + use windmill_common::materialization::{ + MaterializationStatus, RecordMaterializationRequest, UNPARTITIONED, + }; + // A DEPLOYED script only. The annotation is a deploy-time contract — `manual`, + // a three-segment relation, a configured warehouse — checked in + // `create_script_internal`, which also required write access to the path. A + // preview, hub or inline-flow body reaches this function without any of that, + // so honouring it there would let `jobs:run` alone restamp any relation's last + // writer from a script that never touched it. + if job.kind != JobKind::Script { + return; + } + // Cheap guard: the annotation scan is skipped for the overwhelming majority + // of jobs, which carry no `materialize` line at all. + if !code.contains("materialize") { + return; + } + let Some(m) = windmill_parser::asset_parser::parse_pipeline_annotations(code) + .materialize + .filter(|m| m.target_kind == windmill_parser::asset_parser::AssetKind::Dbt) + else { + return; + }; + // The slice this run wrote, resolved once upstream (`resolve_partition_for_job`) + // and carried in the args the cascade reads too, so a partitioned producer + // records the same identity everything else propagates. + let partition = job + .args + .as_ref() + .and_then(|a| a.0.get(windmill_common::partition::PARTITION_ARG)) + .and_then(|v| serde_json::from_str::(v.get()).ok()) + .unwrap_or_else(|| UNPARTITIONED.to_string()); + let (status, error) = match result { + Ok(_) => (MaterializationStatus::Materialized, None), + Err(e) => (MaterializationStatus::Failed, Some(e.to_string())), + }; + let recorded = match conn { + Connection::Sql(db) => windmill_common::materialization::record_materialization( + db, + &job.workspace_id, + windmill_common::assets::AssetKind::Dbt, + &m.target_path, + &partition, + status, + None, + None, + Some(job.id), + error.as_deref(), + ) + .await + .map_err(|e| anyhow::anyhow!("{e:#}")), + Connection::Http(client) => { + crate::agent_workers::record_materialization_from_agent_http( + client, + &job.workspace_id, + &RecordMaterializationRequest { + asset_kind: windmill_common::assets::AssetKind::Dbt, + asset_path: m.target_path.clone(), + partition, + status, + snapshot_id: None, + row_count: None, + job_id: Some(job.id), + error, + schema: None, + }, + ) + .await + } + }; + if let Err(e) = recorded { + tracing::warn!( + "recording the materialization of dbt://{} failed: {e:#}", + m.target_path + ); + } } /// True when `path` contains only `Normal`/`CurDir` components, i.e. it cannot diff --git a/docs/dbt-runtime.md b/docs/dbt-runtime.md index dc06a22fa0..3749bda3ea 100644 --- a/docs/dbt-runtime.md +++ b/docs/dbt-runtime.md @@ -47,6 +47,7 @@ the dominant way dbt is orchestrated today. | 22 | Naming | Match Cosmos field names; importer deferred | | 23 | Descriptor | `wm_dbt.yaml` inside the project, OPTIONAL. See below | | 24 | Warehouse | Configured on the workspace by name, `main` by default. See below | +| 25 | Cascade direction | Into a relation, not out of a run: `// materialize manual dbt://…` declares a write from any language but dbt's own and wakes `# on dbt://…` subscribers; a finished dbt run still does not dispatch. See "No cascade *from* dbt" | ## Decision 1: engine toggle, and why the shipped default is not Fusion yet @@ -157,11 +158,15 @@ build and an enterprise build whose key did not verify. workspace warehouse's NAME, so two scripts running against the same warehouse agree on identity. -The SCHEME names the producer, because dbt is the only thing that creates one of -these: no other language derives warehouse relations, `// materialize` takes -DuckLake targets only, and a dbt run does not dispatch. Calling the kind -something generic promised a parity with native Snowflake and BigQuery scripts -that does not exist. +The SCHEME names the namespace dbt made, not an exclusive producer. dbt is what +put warehouse relations in the asset graph and is what derives them from a +project; no other language *infers* one, and calling the kind something generic +promised a parity with native Snowflake and BigQuery scripts that does not exist. +A script can nonetheless DECLARE that it writes one — `// materialize manual +dbt:////`, in any language but dbt's own, whose writes +come from its manifest — and that declaration lands on the same node the dbt model +reading the relation does, because identity is the relation rather than the tool. +See "No cascade *from* dbt" below. The PATH is the physical relation, and that is the load-bearing half. dbt-core has no cross-project `ref()`: two projects meet when one materializes a mart and @@ -648,10 +653,13 @@ Two consequences worth knowing: dropped would be filtered out of its own run's graph. The pinned version's nodes are the scope instead. -## No cascade from dbt, and no pipeline membership +## No cascade *from* dbt, and no pipeline membership A finished dbt run does not trigger anything. Its models are recorded, drawn and -tracked; they do not fan out. +tracked; they do not fan out. The opposite direction does: a script that declares +`// materialize manual dbt:////` is an ordinary producer +of that relation, and its completion wakes `# on dbt://` subscribers +through the same fan-out every other asset kind uses. A dbt script is also not a pipeline member (`in_pipeline` is forced false for `ScriptLang::Dbt` at deploy). It materializes warehouse tables, so it looks like @@ -663,22 +671,99 @@ Its models are `dbt://` assets in the shared graph regardless: that is what puts a native script reading one of them on the same node, and it is independent of pipeline membership. -dbt already orders its own DAG, so a cascade would only ever add one thing: -waking a Windmill script that reads a mart. That edge is real but narrow, and -only half of it exists — nothing outside dbt can declare a `dbt://` write -(`// materialize` accepts DuckLake targets only), so the reverse direction, an -ingestion script waking a dbt project, cannot be expressed at all. - -Against that, dispatching correctly from dbt is not cheap. A run's `select` can -build any subset of the project, so the deploy-time write set is not what ran; -using it wakes consumers of relations the run never touched, and narrowing it -needs a per-job record of what was built, which the per-relation state table -cannot supply (it keeps one row per relation, stamped with the last writer). +dbt already orders its own DAG, so a cascade out of a run would only ever add one +thing: waking a Windmill script that reads a mart. That edge is real but narrow, +and dispatching it correctly is not cheap. A run's `select` can build any subset +of the project, so the deploy-time write set is not what ran; using it wakes +consumers of relations the run never touched, and narrowing it needs a per-job +record of what was built, which the per-relation state table cannot supply (it +keeps one row per relation, stamped with the last writer). So dbt materializes and reports, and `asset_dispatch` returns early for -`ScriptLang::Dbt`. A `# on dbt://` subscription is refused outright at -deploy rather than accepted and left dormant — an edge drawn on the canvas that -can never fire is worse than an error saying so. +`ScriptLang::Dbt`. Wiring it up later means deciding what a selective run should +notify — that decision is the work, not the plumbing. + +### Declaring the write, and which subscriptions are refused + +`// materialize manual dbt:////` is how an ingestion +script says it writes a warehouse relation. `manual` is not a mode but the only +mode: nothing generates warehouse DDL, so the script issues its own write and +Windmill records the outcome — the same `materialized_partition` row a DuckLake +target lands, so the relation carries a last writer on the run page and the graph. +It is language-agnostic (the DuckLake write ENGINE is DuckDB's; this declaration +is anyone's but a dbt project's, whose writes are read from its manifest), and the +recording happens in the generic job path +(`record_declared_warehouse_write`) rather than in an executor, for the same +reason. Identity is unchanged — the physical relation — so the ingestion script +and the dbt model reading it are one node, and a `source` declared on the relation +puts the whole thing on one lineage. `// data_test` is refused beside it: those +checks are probes the DuckDB executor splices around a managed write, so on a +warehouse relation — which the script writes itself, from any language — nothing +would run them, and a declarer would deploy green with its assertions silently +skipped. Assert on the relation with a dbt test in the project that reads it. +The `` segment is resolved at +deploy for the same reason a descriptor's `profile.warehouse` is: a name no +warehouse answers to is not a namespace, it strands the write on a node nothing +else reaches. + +Known boundary, shared with every other runtime pipeline annotation: the record +is written from the normal execution path, and recording and cascading are decided +separately, so the routes off it differ. + +* A **dedicated worker** never enters that path — it bypasses the record exactly + as it bypasses `// partitioned` resolution — while its job is still a top-level + `Script`, so the fan-out (which reads the deploy-time `asset` rows) runs. It + cascades and records nothing, leaving the relation with no last writer. +* A **flow runner** bypasses the path too, and is routed by `flow_step_id`, which + `is_eligible_kind` rejects. Neither record nor cascade. +* A **flow step running a deployed script** enters the path as a `Script` job, so + it records — and carries a `flow_step_id`, so it never cascades. +* A **flow step with an inline body** is a `FlowScript` job, which the recording + guard excludes along with previews: neither. + +Fixing the recording half is one change for every runtime pipeline annotation, +not this one. + +A `# on dbt://` subscription is held to the same relation a producer +is — a whole `//` under a configured warehouse, checked +by the validator the `// materialize` target goes through, since two spellings of +that rule would refuse and accept the same string. Beyond that it is refused in +exactly one shape: when every script that writes that relation is a dbt one. Nothing +produces it yet is NOT that shape — a subscriber may be deployed before its +producer, as for every other asset kind, and refusing there would break +deploy-order-independent syncs. A dbt script may neither subscribe nor declare a +`// materialize`: its graph ingest republishes that path's trigger and asset rows +wholesale, so either annotation would deploy something the dependency job then +silently removes — while the declared write would still stamp the relation on +every run. + +The producer set is read as it stands committed, minus the deploying script's own +rows — those describe the version being replaced, so a script dropping its +`// materialize` while adding a subscription would otherwise count itself as the +producer that wakes it, which it could not be anyway (the dispatcher skips +self-loops). + +What that leaves is a subscription accepted while it was live and later orphaned. +A dbt project that claims the relation afterwards names those edges in its own log +rather than leaving them silently dormant — the same "an edge that can never fire +is worse than saying so" the refusal is for, at the other point where it is +knowable. Both points that publish ownership warn: the deploy, and a run whose +static descriptor found its profile moved. An agent run publishes none — it is +forced to per-run models, so it stores a job-pinned snapshot and leaves workspace +ownership with the deployed graph — so it cannot orphan a subscription either. + +Two orphanings are reported nowhere, and both are accepted rather than overlooked. +A native producer that drops its `// materialize` and leaves dbt alone on the +relation: the deploy that causes it does not touch the subscriber. And the +interleaving where a dbt ingest commits between a subscriber's producer check and +its own commit — the check sees no producer and accepts, the ingest's warning +query sees no trigger and says nothing. Closing the second means a per-relation +lock shared by the deploy path and the ingest, and the ingest takes +`script … FOR UPDATE` before its own advisory lock, so a deploy holding relation +locks first inverts that order into a deadlock across two subsystems — a worse +failure than the cosmetic edge it would prevent. Both are bounded the same way: +the next deploy of that project warns, and the canvas is where they show +meanwhile. A plain READ still renders the consumer beside the model, which is what makes the lineage one graph — but it is written in the script's own code, not in a @@ -687,8 +772,8 @@ comment: the body parsers resolve an asset URI from a string literal Python, TS/Bun/Deno, DuckDB or Ansible script is the read. Those four are the languages with a body-asset parser; the native warehouse ones (snowflake, bigquery, postgresql, mysql, mssql) declare no assets at all today, so a mart -they consume joins the graph only once that inference exists. Wiring the trigger up later means deciding what a -selective run should notify — that decision is the work, not the plumbing. +they consume joins the graph only once that inference exists — while a relation +one of them WRITES joins it now, through the annotation. ## Live per-model progress, and why only dbt-core 1.x has it @@ -1215,17 +1300,22 @@ Against a real dbt project (jaffle_shop shape) and the local Postgres: script reading one of the marts gets an edge to it. 6. **Shared node**: a native script that READS a mart renders as a reader of the same node the dbt model writes — one node, not two islands. Declared with a - plain read (`# dbt://`), never `# on`: a `dbt://` subscription is - refused at deploy, because nothing but dbt writes a warehouse relation and a - dbt run does not dispatch (see "no cascade from dbt"). -7. **Selection**: descriptor `select`/`exclude`, and a run-arg override, each + plain read (`# dbt://`), never `# on`: a subscription to a relation dbt + alone builds is refused at deploy, since a dbt run does not dispatch. +7. **Declared write**: a native `// materialize manual dbt://` script + and a dbt project reading that relation as a `source` render as one node; a + run of the script records its materialization and wakes a + `# on dbt://` subscriber — a subscription only that producer makes + wakeable, the dbt project reading the relation being no producer of it (see + "no cascade *from* dbt"). +8. **Selection**: descriptor `select`/`exclude`, and a run-arg override, each build only the expected subset. -8. **Dynamic descriptors**: a `{{ }}` placeholder in `vars` re-ingests the graph +9. **Dynamic descriptors**: a `{{ }}` placeholder in `vars` re-ingests the graph from the run's own manifest, so a model that placeholder enables appears in the same run that builds it. -9. **Both credential paths**: resource-rendered `profiles.yml`, and the project's +10. **Both credential paths**: resource-rendered `profiles.yml`, and the project's own `profiles.yml` with env-var injection. -10. **Caching**: a second run reuses the cached `dbt_packages/` with no network +11. **Caching**: a second run reuses the cached `dbt_packages/` with no network fetch. Keep only tests that pin behavior a future change could break. Per AGENTS.md, diff --git a/docs/ducklake-materialization.md b/docs/ducklake-materialization.md index 32fff7ca23..834b48c211 100644 --- a/docs/ducklake-materialization.md +++ b/docs/ducklake-materialization.md @@ -606,8 +606,10 @@ how-to (extract-engine choice, cursor recipes, schema-drift handling, worked examples) lives in windmilldocs `core_concepts/63_pipelines` → "Ingestion (EL)"; this section records only what future feature work must not break. -- **`// materialize` is DuckDB-only** (deploy-rejected elsewhere, managed and - `manual` alike — `windmill-api-scripts/src/scripts.rs`), and the SDK +- **A `ducklake://` `// materialize` is DuckDB-only** (deploy-rejected + elsewhere, managed and `manual` alike — `windmill-api-scripts/src/scripts.rs`; + a `dbt://` warehouse-relation target is the one any language but dbt's own may + declare, and it is track-only — see `docs/dbt-runtime.md`), and the SDK materialize helpers (`upsert_partition` / `upsertPartition`) build their SQL inside the SDK, so the asset parsers cannot see the write. A polyglot node that "writes the lake directly" therefore deploys with **no output edge** — diff --git a/frontend/src/lib/components/assets/AssetGraph/resolveGraph.test.ts b/frontend/src/lib/components/assets/AssetGraph/resolveGraph.test.ts index de00aef2be..c57687d78b 100644 --- a/frontend/src/lib/components/assets/AssetGraph/resolveGraph.test.ts +++ b/frontend/src/lib/components/assets/AssetGraph/resolveGraph.test.ts @@ -319,6 +319,18 @@ describe('resolveGraph', () => { expect(assetTrigKeys(r, 'f/x/open')).toEqual(['ducklake:main.orders']) }) + // A relation a native `// materialize manual dbt://…` script writes wakes its + // subscribers, so hiding the edge would leave the author's own annotation off + // the canvas. The deploy refuses the ones that cannot fire. + it('draws an explicit dbt:// subscription as an unsaved trigger overlay', () => { + const liveAnnotations = { + scriptPath: 'f/x/open', + annotations: ann({ triggerAssets: [{ kind: 'dbt', path: 'main/analytics/orders' }] }) + } + const r = resolveGraph(input({ liveAnnotations })) + expect(assetTrigKeys(r, 'f/x/open')).toEqual(['dbt:main/analytics/orders']) + }) + it('open-script live annotations add unsaved triggers, deduped vs persisted', () => { const base = baseGraph({ triggers: [ diff --git a/frontend/src/lib/components/assets/AssetGraph/resolveGraph.ts b/frontend/src/lib/components/assets/AssetGraph/resolveGraph.ts index 867fa6f036..5c8aa697a4 100644 --- a/frontend/src/lib/components/assets/AssetGraph/resolveGraph.ts +++ b/frontend/src/lib/components/assets/AssetGraph/resolveGraph.ts @@ -77,23 +77,15 @@ function persistedNativeKinds(base: AssetGraphResponse, path: string): Set = new Set(['ducklake', 's3object']) -/** Whether a subscription on this kind can ever fire once deployed. - * - * A `dbt://` one cannot: dbt is the only producer of a warehouse relation and - * a dbt run does not dispatch, so the deploy refuses `// on dbt://…` outright - * (`scripts.rs`). The editor must not draw an arrow the deploy will reject — - * applied to the EXPLICIT overlays; auto-derivation is already scoped by - * `AUTO_TRIGGER_KINDS`. Parsing is left alone so the Rust-parity test still - * compares like for like. */ -function canTrigger(kind: AssetKind): boolean { - return kind !== 'dbt' -} - /** `kind:path` refs of a script's `// materialize` write target(s) (base + * the scd2 `_current` companion), which the body `SELECT` doesn't express. */ function materializeWriteRefs(parsed: PipelineAnnotations): string[] { @@ -381,7 +373,7 @@ function makeContext(input: ResolveGraphInput): ResolveContext { const liveRefKeys = new Set() if (openIsSavedEdit) { if (liveAnnotations.scriptPath === openPath) { - for (const a of liveAnnotations.annotations.triggerAssets.filter((a) => canTrigger(a.kind))) + for (const a of liveAnnotations.annotations.triggerAssets) liveRefKeys.add(`${a.kind}:${a.path}`) // The `// materialize ` target is a declared *output*, but it // lives in an annotation (not the SQL body), so neither triggerAssets @@ -625,7 +617,7 @@ function seedDraftOverlays(acc: Accumulator, input: ResolveGraphInput) { // stable when the user clicks off this draft. Live annotations // (below) take over for the currently-open draft so keystroke // edits still update in real time. - for (const a of parsed.triggerAssets.filter((a) => canTrigger(a.kind))) { + for (const a of parsed.triggerAssets) { extraTriggers.push({ trigger_kind: 'asset', asset_kind: a.kind, @@ -704,7 +696,7 @@ function applyLiveBufferOverlay(acc: Accumulator, input: ResolveGraphInput, ctx: for (let i = extraTriggers.length - 1; i >= 0; i--) { if (extraTriggers[i].runnable_path === livePath) extraTriggers.splice(i, 1) } - for (const a of liveAnnotations.annotations.triggerAssets.filter((a) => canTrigger(a.kind))) { + for (const a of liveAnnotations.annotations.triggerAssets) { const key = `${a.kind}:${a.path}` if (assetKeys.has(key)) continue extraTriggers.push({ diff --git a/frontend/src/lib/components/assets/lib.ts b/frontend/src/lib/components/assets/lib.ts index 23531d71f2..93367a725e 100644 --- a/frontend/src/lib/components/assets/lib.ts +++ b/frontend/src/lib/components/assets/lib.ts @@ -96,10 +96,11 @@ export function formatAssetKind(asset: { case 'volume': return 'Volume' case 'dbt': - // The SCHEME says dbt because dbt is the only thing that creates one; - // the PATH stays the relation, so a mart one project builds and the - // `source` the next reads land on one node — their dbt `unique_id`s - // differ where the relation does not (docs/dbt-runtime.md, decision 11). + // The SCHEME says dbt because dbt is what derives these relations; the + // PATH stays the relation, so a mart one project builds, the `source` + // the next reads, and a native `// materialize manual dbt://…` writer + // land on one node — their dbt `unique_id`s differ where the relation + // does not (docs/dbt-runtime.md, decision 11). return 'dbt table' } } diff --git a/frontend/src/lib/components/copilot/chat/pipeline/core.ts b/frontend/src/lib/components/copilot/chat/pipeline/core.ts index 7583c1b356..f696b1c869 100644 --- a/frontend/src/lib/components/copilot/chat/pipeline/core.ts +++ b/frontend/src/lib/components/copilot/chat/pipeline/core.ts @@ -338,8 +338,8 @@ export function getPipelinePromptSection(ctx: PipelineContext): string { Data Pipeline editor (ACTIVE): - The user has the /pipeline/${ctx.folder} editor open. A pipeline is a DAG of scripts (nodes) connected by storage assets (DuckLake tables, data tables, S3 objects, volumes, resources) and execution triggers. - Annotations are top-of-file comments in the NODE'S OWN comment syntax: \`--\` for SQL (duckdb/postgresql), \`#\` for python3/bash, \`//\` for bun/TS. The \`//\` shown below is the TS form — translate it (a \`// pipeline\` line in a SQL node is a syntax error that won't deploy). -- A script becomes a pipeline node when its source starts with the \`// pipeline\` annotation. Declare execution-DAG inputs with \`// on \` (e.g. \`// on ducklake://main/orders\`). Outputs are inferred from what the body writes (wmill SDK calls / SQL CREATE TABLE / writeS3File); declare a managed output with \`// materialize \`. Optional badges: \`// partitioned \`, \`// freshness \`, \`// tag \`, \`// retry [delay]\`, \`// data_test ...\`, \`// measure = [where ]\`, \`// dimension = \`. -- \`materialize\` (the managed output): \`// materialize \` means the runtime writes the node's output table FOR you — write the body as a single SELECT and the runtime wraps it in the create/replace, so do NOT also write your own CREATE TABLE / INSERT. IMPORTANT: \`// materialize\` is **DuckDB-only** and its target MUST be a DuckLake table (\`ducklake:///
\`) — deploy rejects it on any other language or target. For a \`python3\`/\`bun\`/\`postgresql\` node, do NOT use \`// materialize\`; write the output via the SDK instead (e.g. \`wmill.writeS3File(...)\`, a \`CREATE TABLE\` in postgresql, or \`wmill.databaseUrlFromResource\`/ducklake helpers) and let the output be inferred. Reach for \`duckdb\` when a node should materialize a DuckLake table. Write strategy: with no option it REPLACES the whole table each run (full refresh; the only mode whose output columns may change); \`// materialize append\` INSERT-appends rows (incremental); \`// materialize key=\` merges/upserts on \`\`. \`// materialize manual \` opts OUT of managed writes — the script writes its own DDL and the annotation only records the output asset for lineage. \`materialize\` is paired with partitioning for incremental pipelines: a \`// partitioned \` node runs once per partition (append/merge into a fixed-schema table), and the \`{partition}\` token — usable in any asset URI AND in the body SQL — is substituted with the current partition's IDENTITY string at run time. To filter the source to the active slice on a time grain, use the runtime-injected macro: \`WHERE wm_partition() = {partition}\`. \`wm_partition(ts)\` buckets a timestamp with the exact identity format the runtime used (daily/hourly/weekly/monthly), so it always matches and you never hand-write a \`strftime\` format. Do NOT write \`= TIMESTAMP {partition}\`: the identity string is not a valid timestamp literal for hourly/weekly/monthly and errors at runtime. For \`dynamic\` partitioning the identity is your caller-supplied key (not a timestamp, no macro), so filter on it directly: \`WHERE = {partition}\`. \`materialize\` is an output DECLARATION on the node — it is not a command; there is no "materialize run". +- A script becomes a pipeline node when its source starts with the \`// pipeline\` annotation. Declare execution-DAG inputs with \`// on \` (e.g. \`// on ducklake://main/orders\`). Outputs are inferred from what the body writes (wmill SDK calls / SQL CREATE TABLE / writeS3File); declare a managed output with \`// materialize \`. Optional badges: \`// partitioned \`, \`// freshness \`, \`// tag \`, \`// retry [delay]\`, \`// data_test ...\` (managed DuckLake targets only — deploy rejects it beside a \`dbt://\` target), \`// measure = [where ]\`, \`// dimension = \`. +- \`materialize\` (the managed output): a managed \`// materialize ducklake:///
\` means the runtime writes the node's output table FOR you — write the body as a single SELECT and the runtime wraps it in the create/replace, so do NOT also write your own CREATE TABLE / INSERT. The \`dbt://\` target below is the opposite: the node writes its own DDL and none of the write strategies apply to it. IMPORTANT: a MANAGED \`// materialize\` is **DuckDB-only** and its target MUST be a DuckLake table (\`ducklake:///
\`) — deploy rejects a \`ducklake://\` target on any other language. For a \`python3\`/\`bun\`/\`postgresql\` node writing the lake, do NOT use \`// materialize\`; write the output via the SDK instead (e.g. \`wmill.writeS3File(...)\`, a \`CREATE TABLE\` in postgresql, or \`wmill.databaseUrlFromResource\`/ducklake helpers) and let the output be inferred. Reach for \`duckdb\` when a node should materialize a DuckLake table. The one target any language BUT DBT'S OWN may declare (a dbt project's writes come from its manifest, so \`// materialize\` on a dbt script is rejected at deploy) is a WAREHOUSE RELATION: \`// materialize manual dbt:////\`, with \`\` a warehouse the workspace configures under Settings → dbt. \`manual\` is its only mode — nothing generates warehouse DDL, so the node issues its own write and the annotation records the outcome. Use it on an ingestion node a dbt project reads as a \`source\`: the declared relation and the dbt model become ONE graph node, and a downstream \`// on dbt:////\` fires when that node completes. Write strategy: with no option it REPLACES the whole table each run (full refresh; the only mode whose output columns may change); \`// materialize append\` INSERT-appends rows (incremental); \`// materialize key=\` merges/upserts on \`\`. \`// materialize manual \` opts OUT of managed writes — the script writes its own DDL and the annotation only records the output asset for lineage. \`materialize\` is paired with partitioning for incremental pipelines: a \`// partitioned \` node runs once per partition (append/merge into a fixed-schema table), and the \`{partition}\` token — usable in any asset URI AND in the body SQL — is substituted with the current partition's IDENTITY string at run time. To filter the source to the active slice on a time grain, use the runtime-injected macro: \`WHERE wm_partition() = {partition}\`. \`wm_partition(ts)\` buckets a timestamp with the exact identity format the runtime used (daily/hourly/weekly/monthly), so it always matches and you never hand-write a \`strftime\` format. Do NOT write \`= TIMESTAMP {partition}\`: the identity string is not a valid timestamp literal for hourly/weekly/monthly and errors at runtime. For \`dynamic\` partitioning the identity is your caller-supplied key (not a timestamp, no macro), so filter on it directly: \`WHERE = {partition}\`. \`materialize\` is an output DECLARATION on the node — it is not a command; there is no "materialize run". - \`measure\` / \`dimension\` (declared metrics): on a node that materializes a DuckLake table, \`// measure = [where ]\` names the canonical way to aggregate that table (e.g. \`// measure revenue = sum(amount) where not is_refund\`), and \`// dimension = \` names a way to slice it (e.g. \`// dimension region = region\`, \`// dimension month = date_trunc('month', ordered_at)\`). They execute nothing: they are catalogued at deploy so the editor and other agents can reuse the definition instead of re-deriving it and silently disagreeing. Keep the predicate in the \`where\` clause rather than folding it into the aggregate: it is rendered as \` FILTER (WHERE )\`, which is what lets two measures with different predicates sit under one GROUP BY. DuckLake-only, and only meaningful next to \`// materialize\`. Declare one when a number carries a judgement call someone else would get wrong (refunds excluded, test rows dropped, which column is the amount); do NOT blanket every table with measures, an obvious \`count(*)\` earns nothing. To USE a metric another node declares, read that node with read_pipeline_node and reuse its exact expression rather than guessing it. - Use get_pipeline_graph to see the current nodes/assets/triggers, and read_pipeline_node before editing one. - Every node of this pipeline lives at \`f/${ctx.folder}/\` — \`${ctx.folder}\` is the folder name and \`f/\` is the owner prefix every workspace path carries, so write it exactly once (never \`f/f/…\`, and never a bare \`\`). diff --git a/system_prompts/auto-generated/prompts.ts b/system_prompts/auto-generated/prompts.ts index 1feeadc162..5776fd6d0d 100644 --- a/system_prompts/auto-generated/prompts.ts +++ b/system_prompts/auto-generated/prompts.ts @@ -984,7 +984,7 @@ A script joins the pipeline when its source begins with the \`pipeline\` annotat SELECT * FROM read_csv($file) \`\`\` - **Outputs** are inferred from what the body writes — a \`CREATE TABLE\`, a \`wmill.writeS3File(...)\`, a DuckLake/datatable write. To declare a managed output explicitly, use \`// materialize \`. -- Optional badges: \`// partitioned \`, \`// freshness \` (e.g. \`1h\`), \`// tag \`, \`// retry [delay]\`, \`// data_test ...\`. +- Optional badges: \`// partitioned \`, \`// freshness \` (e.g. \`1h\`), \`// tag \`, \`// retry [delay]\`, \`// data_test ...\` (managed DuckLake targets only — deploy rejects it beside a \`dbt://\` one). ## S3 object wiring (storage form matters) @@ -999,9 +999,11 @@ The key must be a **string literal** — the graph parser is static and cannot f ## Materialize (the managed output) -> **\`// materialize\` is DuckDB-only**, and its target must be a DuckLake table (\`ducklake:///
\`). Deploy **rejects** \`// materialize\` on any other language (\`python3\`, \`bun\`, \`postgresql\`) or a non-DuckLake target. For a non-DuckDB node, do **not** use \`// materialize\` — write the output via the SDK (\`wmill.writeS3File(...)\`, a postgresql \`CREATE TABLE\`, ducklake helpers, …) and let it be inferred. Use \`duckdb\` when a node should materialize a DuckLake table. +> **A MANAGED \`// materialize\` is DuckDB-only**, and its target must be a DuckLake table (\`ducklake:///
\`). Deploy **rejects** a \`ducklake://\` \`// materialize\` on any other language (\`python3\`, \`bun\`, \`postgresql\`). For a non-DuckDB node writing the lake, do **not** use \`// materialize\` — write the output via the SDK (\`wmill.writeS3File(...)\`, ducklake helpers, …) and let it be inferred. Use \`duckdb\` when a node should materialize a DuckLake table. +> +> The one target ANY language may declare (except a dbt script, whose writes come from its manifest) is a **warehouse relation**: \`// materialize manual dbt:////\`, where \`\` is a warehouse the workspace configures under Settings → dbt. \`manual\` is the only mode it has — nothing generates warehouse DDL, so the node issues its own write (a postgresql \`CREATE TABLE\` / \`INSERT\`, an SDK load, …) and the annotation records the outcome. Use it on an ingestion node whose output a dbt project reads as a \`source\`: the declared relation and the dbt model land on ONE graph node, and a downstream \`// on dbt:////\` fires when the ingestion node completes. -\`// materialize \` tells the runtime to write the node's output table **for you**: write the body as a single \`SELECT\` and the runtime wraps it in the create/replace — do **not** also write your own \`CREATE TABLE\` / \`INSERT\`. Write strategy: +A managed \`// materialize ducklake:///
\` tells the runtime to write the node's output table **for you**: write the body as a single \`SELECT\` and the runtime wraps it in the create/replace — do **not** also write your own \`CREATE TABLE\` / \`INSERT\`. (The opposite holds for the \`dbt://\` target above: there the node writes its own DDL and the strategies below do not apply.) Write strategy: - no option → **replace** the whole table each run (full refresh; the only mode whose output columns may change); - \`// materialize append\` → INSERT-append rows (incremental); diff --git a/system_prompts/base/pipeline-base.md b/system_prompts/base/pipeline-base.md index b2b428c401..3f2d985de4 100644 --- a/system_prompts/base/pipeline-base.md +++ b/system_prompts/base/pipeline-base.md @@ -37,7 +37,7 @@ A script joins the pipeline when its source begins with the `pipeline` annotatio SELECT * FROM read_csv($file) ``` - **Outputs** are inferred from what the body writes — a `CREATE TABLE`, a `wmill.writeS3File(...)`, a DuckLake/datatable write. To declare a managed output explicitly, use `// materialize `. -- Optional badges: `// partitioned `, `// freshness ` (e.g. `1h`), `// tag `, `// retry [delay]`, `// data_test ...`. +- Optional badges: `// partitioned `, `// freshness ` (e.g. `1h`), `// tag `, `// retry [delay]`, `// data_test ...` (managed DuckLake targets only — deploy rejects it beside a `dbt://` one). ## S3 object wiring (storage form matters) @@ -52,9 +52,11 @@ The key must be a **string literal** — the graph parser is static and cannot f ## Materialize (the managed output) -> **`// materialize` is DuckDB-only**, and its target must be a DuckLake table (`ducklake:///
`). Deploy **rejects** `// materialize` on any other language (`python3`, `bun`, `postgresql`) or a non-DuckLake target. For a non-DuckDB node, do **not** use `// materialize` — write the output via the SDK (`wmill.writeS3File(...)`, a postgresql `CREATE TABLE`, ducklake helpers, …) and let it be inferred. Use `duckdb` when a node should materialize a DuckLake table. +> **A MANAGED `// materialize` is DuckDB-only**, and its target must be a DuckLake table (`ducklake:///
`). Deploy **rejects** a `ducklake://` `// materialize` on any other language (`python3`, `bun`, `postgresql`). For a non-DuckDB node writing the lake, do **not** use `// materialize` — write the output via the SDK (`wmill.writeS3File(...)`, ducklake helpers, …) and let it be inferred. Use `duckdb` when a node should materialize a DuckLake table. +> +> The one target ANY language may declare (except a dbt script, whose writes come from its manifest) is a **warehouse relation**: `// materialize manual dbt:////`, where `` is a warehouse the workspace configures under Settings → dbt. `manual` is the only mode it has — nothing generates warehouse DDL, so the node issues its own write (a postgresql `CREATE TABLE` / `INSERT`, an SDK load, …) and the annotation records the outcome. Use it on an ingestion node whose output a dbt project reads as a `source`: the declared relation and the dbt model land on ONE graph node, and a downstream `// on dbt:////` fires when the ingestion node completes. -`// materialize ` tells the runtime to write the node's output table **for you**: write the body as a single `SELECT` and the runtime wraps it in the create/replace — do **not** also write your own `CREATE TABLE` / `INSERT`. Write strategy: +A managed `// materialize ducklake:///
` tells the runtime to write the node's output table **for you**: write the body as a single `SELECT` and the runtime wraps it in the create/replace — do **not** also write your own `CREATE TABLE` / `INSERT`. (The opposite holds for the `dbt://` target above: there the node writes its own DDL and the strategies below do not apply.) Write strategy: - no option → **replace** the whole table each run (full refresh; the only mode whose output columns may change); - `// materialize append` → INSERT-append rows (incremental); From 8f553eab353103fd8a28a00532e1766f133590de Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 7 Sep 2026 15:38:38 +0000 Subject: [PATCH 08/15] fix: point the app viewer's edit button at the editor for the app's kind (#11009) Claude-Session: https://claude.ai/code/session_01GDiZaPzhC4R9G4hLPgy1B2 Co-authored-by: Claude Opus 5 (1M context) --- .../apps/editor/InWorkspaceAppViewer.svelte | 16 +++++++++++----- .../components/search/GlobalSearchModal.svelte | 2 +- .../(logged)/apps/get/[...path]/+page.svelte | 10 +++++----- .../(logged)/apps_raw/get/[...path]/+page.svelte | 3 +-- 4 files changed, 18 insertions(+), 13 deletions(-) diff --git a/frontend/src/lib/components/apps/editor/InWorkspaceAppViewer.svelte b/frontend/src/lib/components/apps/editor/InWorkspaceAppViewer.svelte index d5839c6766..99ea07c8fd 100644 --- a/frontend/src/lib/components/apps/editor/InWorkspaceAppViewer.svelte +++ b/frontend/src/lib/components/apps/editor/InWorkspaceAppViewer.svelte @@ -22,19 +22,22 @@ let { workspace, - path, - editHref + path }: { workspace: string path: string - /** Where the Edit button points (low-code vs raw editor). */ - editHref: string } = $props() let app: any = $state(undefined) let notExists = $state(false) let noPermission = $state(false) let canWriteApp = $state(false) + /** Raw vs low-code, read from the app itself rather than from the route: + * both kinds render here and either route serves either kind (links to a raw + * app point at /apps/get all over the app), so only the app can say which + * editor the Edit button must open. */ + let isRawApp = $state(false) + let editHref = $derived(`${base}/${isRawApp ? 'apps_raw' : 'apps'}/edit/${path}?nodraft=true`) let refresh: (() => void) | undefined // The opaque iframe loads the dedicated cookieless, chrome-less viewer route. @@ -103,11 +106,14 @@ } } - // Edit button: determine write access on this real-origin page (cookie). + // Edit button: determine write access and which editor to open on this + // real-origin page (cookie). The sandboxed low-code app never loads on this + // page (it loads inside the opaque iframe), so `app` can't be the source. async function loadPerms() { try { const lite: any = await AppService.getAppLiteByPath({ workspace, path }) canWriteApp = canWrite(lite?.path, lite?.extra_perms ?? {}, $userStore) + isRawApp = !!lite?.raw_app } catch (_) { canWriteApp = false } diff --git a/frontend/src/lib/components/search/GlobalSearchModal.svelte b/frontend/src/lib/components/search/GlobalSearchModal.svelte index a117d2769c..084ad9d492 100644 --- a/frontend/src/lib/components/search/GlobalSearchModal.svelte +++ b/frontend/src/lib/components/search/GlobalSearchModal.svelte @@ -427,7 +427,7 @@ path = `/apps/get/${e.path}` break case 'raw_app': - path = `/raw_apps/get/${e.path}` + path = `/apps_raw/get/${e.path}` break default: path = '/' diff --git a/frontend/src/routes/(root)/(logged)/apps/get/[...path]/+page.svelte b/frontend/src/routes/(root)/(logged)/apps/get/[...path]/+page.svelte index 0af3cdeded..4ff8670cda 100644 --- a/frontend/src/routes/(root)/(logged)/apps/get/[...path]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/apps/get/[...path]/+page.svelte @@ -1,11 +1,11 @@ + + dispatch('canceled')} + on:confirmed={() => dispatch('confirmed', { selectedTriggers, selectedAgents })} +> +
+ {#if draftTriggers.length > 0} +
+
+ {`Your ${runnable} has draft triggers. Select which draft triggers to deploy with the ${runnable}. Undeployed draft triggers will be permanently deleted.`} +
+ +
5 ? 'h-[300px]' : ''}> + +
+ + + + + + + {#each draftTriggers as trigger} + {@const SvelteComponent = triggerIconMap[trigger.type]} + {@const permission = checkSavePermissions(trigger)} + {@const isSelectedTrigger = isSelected(selectedTriggers, trigger)} + + + + + + {/each} + + + + + {/if} + + {#if draftAgents.length > 0} +
+
+ Saved agents this flow uses have unsaved changes. Select which ones to deploy with the + flow. An agent kept as a draft stays editable, and the flow runs the agent as currently + deployed. +
+ +
5 ? 'h-[300px]' : ''}> + +
+ + + + + + + + {#each draftAgents as agent (agent.path)} + {@const permission = checkAgentPermissions(agent)} + {@const isSelectedAgent = selectedAgents.some((a) => a.path === agent.path)} + + + + + + + + {/each} + + + + + {/if} + + diff --git a/frontend/src/lib/components/common/confirmationModal/DraftTriggersConfirmationModal.svelte b/frontend/src/lib/components/common/confirmationModal/DraftTriggersConfirmationModal.svelte deleted file mode 100644 index cfebae0e9f..0000000000 --- a/frontend/src/lib/components/common/confirmationModal/DraftTriggersConfirmationModal.svelte +++ /dev/null @@ -1,168 +0,0 @@ - - - dispatch('canceled')} - on:confirmed={() => dispatch('confirmed', { selectedTriggers })} -> -
-
- {`${isFlow ? 'Your flow' : 'Your script'} has draft triggers. Select which draft triggers to deploy with the ${isFlow ? 'flow' : 'script'}. Undeployed - draft triggers will be permanently deleted.`} -
- -
5 ? 'h-[300px]' : ''}> - -
- - - - - - - {#each draftTriggers as trigger} - {@const SvelteComponent = triggerIconMap[trigger.type]} - {@const permission = checkSavePermissions(trigger)} - {@const isSelectedTrigger = isSelected(selectedTriggers, trigger)} - - - - - - {/each} - - {#if draftTriggers.length === 0} - - - - {/if} - - - - - diff --git a/frontend/src/lib/components/flows/agentDraft.svelte.ts b/frontend/src/lib/components/flows/agentDraft.svelte.ts index 5a6a67ccb8..d4f8fa35de 100644 --- a/frontend/src/lib/components/flows/agentDraft.svelte.ts +++ b/frontend/src/lib/components/flows/agentDraft.svelte.ts @@ -6,6 +6,7 @@ import { sendUserToast } from '$lib/toast' import { canWrite } from '$lib/utils' import { userStore } from '$lib/stores' import { getUserExt } from '$lib/user' +import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte' import { useTriggerDraftSync, type TriggerDraftSync } from '../triggers/useTriggerDraftSync.svelte' import { logReusableAgentUsage } from './agentTelemetry' import { @@ -84,6 +85,95 @@ export interface AgentResourceState { wsSpecific: boolean } +/** + * Why writing this draft to its resource would be refused, if it would. Shared with the flow's + * deploy dialog, which lists the drafts of the agents a flow links: an agent the editor's own + * Deploy button rejects must not be deployable from a flow either. + * + * `currentPath` is the path the draft is being deployed to; pass undefined where the caller has + * none of its own to compare against. + */ +export function agentDraftDeployRefusal( + state: AgentResourceState, + currentPath: string | undefined +): string | undefined { + // The editor offers only values, but a transform can arrive from a step that was forked + // before this existed, or from the generic resource editor: say so rather than writing it. + const transformValued = transformValuedBrainKeys(state.args) + if (transformValued.length > 0) { + const fields = transformValued.map((key) => AGENT_BRAIN_LABELS[key] ?? key) + const many = fields.length > 1 + return `${fields.join(', ')} ${many ? 'are' : 'is'} set to an expression or an AI-filled value, which a saved agent cannot store. Replace ${many ? 'them' : 'it'} with a plain value before deploying.` + } + // The resource endpoint takes any JSON, so nothing downstream stops an agent that cannot run: + // the worker needs a provider to call and rejects a tool whose name it cannot pass to the + // model. Deploying one would break every flow linking it, so it is refused here. + const blocked = agentConfigRunError(state.args) + if (blocked) { + return blocked + } + // Renaming is not the agent editor's to do: moving the resource leaves every step that links to + // it naming a path that no longer exists, and reconciling those is a feature of its own. A + // renamed path can still reach here, the generic editor writing the same draft row and offering + // a path field, so refuse it rather than performing half of a rename. + if (currentPath && state.path !== currentPath) { + return `This draft renames the agent to ${state.path}. Deploy it from the resource editor instead.` + } + // Only a draft naming another type: the load refuses a resource that is not an agent, while a + // draft the generic resource editor wrote names no type at all and inherits the loaded one. + if (state.resource_type && state.resource_type !== 'ai_agent') { + return `This draft is a ${state.resource_type} resource, not an agent.` + } + return undefined +} + +/** + * Write an agent to its resource from the state the editor holds, which can be ahead of the + * persisted draft row: the form stays editable while a deploy is in flight. Surfaces that deploy + * the row itself go through `deployDraft` instead. + * + * `notAnAgent` separates the one failure that invalidates the caller's whole view of the path, its + * holding something else now, from a write that merely failed. + */ +type AgentWriteResult = { ok: true } | { ok: false; error: string; notAnAgent?: true } + +async function writeAgentResource( + workspace: string, + state: AgentResourceState, + noDeployed: boolean +): Promise { + const body = { + path: state.path, + value: state.args, + description: state.description, + labels: state.labels, + ws_specific: state.wsSpecific + } + try { + if (noDeployed) { + // A create needs a type, and every caller proved this path is an agent before offering it. + await ResourceService.createResource({ + workspace, + requestBody: { ...body, resource_type: state.resource_type ?? 'ai_agent' } + }) + } else { + // The type the caller proved is as old as its own load, and an update carries no type of + // its own: were the path deleted and recreated as something else meanwhile, this write + // would put an agent config inside that resource. Reading it again narrows the window to + // the request rather than to however long the editor or the dialog stayed open. + const current = await ResourceService.getResource({ workspace, path: state.path }) + const refused = agentEditorRefusal(state.path, current.resource_type) + if (refused) { + return { ok: false, error: refused, notAnAgent: true } + } + await ResourceService.updateResource({ workspace, path: state.path, requestBody: body }) + } + } catch (err) { + return { ok: false, error: `Could not save agent: ${err}` } + } + return { ok: true } +} + export interface AgentDraftOptions { /** The `ai_agent` resource being edited. */ path: () => string | undefined @@ -219,6 +309,16 @@ export function useAgentDraft(opts: AgentDraftOptions): AgentDraftHandle { // config before the autosave lands. state = ((r as any).draft as AgentResourceState | undefined) ?? structuredClone(deployedState) + // Adopt the row's timestamp as this tab's baseline. Without it the first save from + // each tab goes out with no `last_sync`, which the backend treats as unconditional + // and so silently overwrites another tab's newer draft. It also clears any parked + // conflict or failure for the key: a conflict is deliberately sticky (the retry + // keeps the same baseline), and nothing else mounts a resolver for `resource` + // drafts, so re-opening the agent is the only place it can be resolved. + UserDraftDbSyncer.recordRemoteSync( + { workspace: ws, itemKind: 'resource', path }, + (r as { draft_saved_at?: string }).draft_saved_at + ) loading = false await sync.maybeRestore() }, @@ -237,42 +337,9 @@ export function useAgentDraft(opts: AgentDraftOptions): AgentDraftHandle { const ws = opts.workspace() const s = state if (!ws || !s) return false - // The editor offers only values, but a transform can arrive from a step that was forked - // before this existed, or from the generic resource editor: say so rather than writing it. - const transformValued = transformValuedBrainKeys(s.args) - if (transformValued.length > 0) { - const fields = transformValued.map((key) => AGENT_BRAIN_LABELS[key] ?? key) - const many = fields.length > 1 - sendUserToast( - `${fields.join(', ')} ${many ? 'are' : 'is'} set to an expression or an AI-filled value, which a saved agent cannot store. Replace ${many ? 'them' : 'it'} with a plain value before deploying.`, - true - ) - return false - } - // The resource endpoint takes any JSON, so nothing downstream stops an agent that cannot run: - // the worker needs a provider to call and rejects a tool whose name it cannot pass to the - // model. Deploying one would break every flow linking it, so it is refused here. - const blocked = agentConfigRunError(s.args) - if (blocked) { - sendUserToast(blocked, true) - return false - } - // Renaming is not this editor's to do: moving the resource leaves every step that links to it - // naming a path that no longer exists, and reconciling those is a feature of its own. A - // renamed path can still reach here, the generic editor writing the same draft row and - // offering a path field, so refuse it rather than performing half of a rename. - const currentPath = opts.path() - if (currentPath && s.path !== currentPath) { - sendUserToast( - `This draft renames the agent to ${s.path}. Deploy it from the resource editor instead.`, - true - ) - return false - } - // Only a draft naming another type: the load refuses a resource that is not an agent, while a - // draft the generic resource editor wrote names no type at all and inherits the loaded one. - if (s.resource_type && s.resource_type !== 'ai_agent') { - sendUserToast(`This draft is a ${s.resource_type} resource, not an agent.`, true) + const refused = agentDraftDeployRefusal(s, opts.path()) + if (refused) { + sendUserToast(refused, true) return false } // The form stays editable while the request is in flight, so everything below works from a @@ -280,39 +347,15 @@ export function useAgentDraft(opts: AgentDraftOptions): AgentDraftHandle { // made during the request as saved, and the banner would clear on a value the server never // received; against the snapshot it stays a draft, which is what it is. const submitted = structuredClone($state.snapshot(s)) as AgentResourceState - const body = { - path: submitted.path, - value: submitted.args, - description: submitted.description, - labels: submitted.labels, - ws_specific: submitted.wsSpecific - } - try { - if (noDeployed) { - await ResourceService.createResource({ - workspace: ws, - // A create needs a type, and the load proved this path is an agent before opening. - requestBody: { ...body, resource_type: submitted.resource_type ?? 'ai_agent' } - }) + const written = await writeAgentResource(ws, submitted, noDeployed) + if (!written.ok) { + // A path that is no longer an agent tears this editor down; anything else is a plain error + // the user can retry from the form as it stands. + if (written.notAnAgent) { + refuse(written.error) } else { - // The type this editor proved is as old as the load, and an update carries no type of - // its own: were the path deleted and recreated as something else meanwhile, this write - // would put an agent config inside that resource. Reading it again narrows the window - // to the request rather than to however long the editor stayed open. - const current = await ResourceService.getResource({ workspace: ws, path: submitted.path }) - const refused = agentEditorRefusal(submitted.path, current.resource_type) - if (refused) { - refuse(refused) - return false - } - await ResourceService.updateResource({ - workspace: ws, - path: submitted.path, - requestBody: body - }) + sendUserToast(written.error, true) } - } catch (err) { - sendUserToast(`Could not save agent: ${err}`, true) return false } // The counter the step card's write-back used to report, from the surface that now owns the diff --git a/frontend/src/lib/components/flows/agentEditorStore.svelte.ts b/frontend/src/lib/components/flows/agentEditorStore.svelte.ts index 83660212b3..025a426d28 100644 --- a/frontend/src/lib/components/flows/agentEditorStore.svelte.ts +++ b/frontend/src/lib/components/flows/agentEditorStore.svelte.ts @@ -6,6 +6,8 @@ * resources page. Module-level rather than a context value because what opens it — a step's card, * a list row — unmounts the moment the selection moves. */ +import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte' + export interface AgentEditorTarget { path: string /** The workspace the opener operates on; the nav workspace when absent. */ @@ -61,3 +63,25 @@ export function markAgentWritten(workspace: string | undefined, path: string) { export function agentWriteCount(workspace: string | undefined, path: string | undefined): number { return agentWrites[writeKey(workspace, path)] ?? 0 } + +/** How many times each agent's DRAFT has been saved, for the surfaces that display an agent by + * fetching it. A draft write moves no deployed version, so `agentWriteCount` never sees it, and a + * card keyed on that alone would keep describing the config a test no longer runs. */ +let agentDraftSaves = $state>({}) + +/** Every writer in this document goes through the draft syncer — this editor, the generic resource + * editor — so one subscription answers for them all, firing when the write lands rather than on + * each keystroke. `resource` is the item kind agent draft rows use. In-memory: a save in another + * tab never arrives here, so a card lags it until reload, while what a test runs is read live. */ +UserDraftDbSyncer.onAnySaved(({ workspace, itemKind, path }) => { + if (itemKind !== 'resource') return + const key = writeKey(workspace, path) + agentDraftSaves[key] = (agentDraftSaves[key] ?? 0) + 1 +}) + +export function agentDraftSaveCount( + workspace: string | undefined, + path: string | undefined +): number { + return agentDraftSaves[writeKey(workspace, path)] ?? 0 +} diff --git a/frontend/src/lib/components/flows/agentTelemetry.ts b/frontend/src/lib/components/flows/agentTelemetry.ts index fcfad440ff..87fcb0dad4 100644 --- a/frontend/src/lib/components/flows/agentTelemetry.ts +++ b/frontend/src/lib/components/flows/agentTelemetry.ts @@ -13,6 +13,10 @@ export type ReusableAgentEvent = | 'linked' /** A linked step was forked back into a standalone agent. */ | 'unlinked' + /** A linked agent's unsaved draft was deployed alongside the flow that uses it. */ + | 'draft_deployed_with_flow' + /** A linked agent's unsaved draft was left as a draft when its flow was deployed. */ + | 'draft_kept_on_deploy' export function logReusableAgentUsage(event: ReusableAgentEvent): void { logFeatureUsage('ai_agent', 'reusable', { key: event }) diff --git a/frontend/src/lib/components/flows/content/AgentEditorHost.svelte b/frontend/src/lib/components/flows/content/AgentEditorHost.svelte index 79db0cedb3..a7c867a6f4 100644 --- a/frontend/src/lib/components/flows/content/AgentEditorHost.svelte +++ b/frontend/src/lib/components/flows/content/AgentEditorHost.svelte @@ -29,6 +29,7 @@ inputTransformsToAgentConfig, type AIAgentConfig } from '../agentResourceUtils' + import { agentArgsToTransforms } from '../linkedAgentDrafts' import { AGENT_TOOLS_ROW } from '../agentFormFields' import { toolDisplayName, type AgentTool } from '../agentToolUtils' import { useAgentDraft } from '../agentDraft.svelte' @@ -223,19 +224,6 @@ }) }) - /** Every argument the resource carries, as a static transform. `tools` is the roster rather than - * a field, so it rides on the module's own key instead. Not only the keys the form renders: a - * run reads them all, and an agent holding its own `user_message` answers with it when nothing - * overrides it, so a test here has to run the configuration a linked step would. */ - function agentArgsToTransforms(args: AIAgentConfig): Record { - const it: Record = {} - for (const [key, value] of Object.entries(args ?? {})) { - if (key === 'tools' || value === undefined) continue - it[key] = { type: 'static', value } as InputTransform - } - return it - } - /** Everything the form does not model. `inputTransformsToAgentConfig` rebuilds the value from * `AGENT_BRAIN_KEYS` alone, so a key this editor never renders — one a newer backend added, or * the `user_message` default a resource may carry, which the runtime does read when the step diff --git a/frontend/src/lib/components/flows/content/AgentEditorModal.svelte b/frontend/src/lib/components/flows/content/AgentEditorModal.svelte index 3ded2deb9f..22ac424144 100644 --- a/frontend/src/lib/components/flows/content/AgentEditorModal.svelte +++ b/frontend/src/lib/components/flows/content/AgentEditorModal.svelte @@ -183,7 +183,9 @@ const moduleIds = new Set(linkedModulesForAgent(scope, path)) moduleIds.add(at.host.moduleId) return Promise.all( - [...moduleIds].map((moduleId) => publishLinkedAgentTools(path, at.ws, scope, moduleId)) + // With the draft: a deploy leaves none, but a version restore leaves the draft standing and + // it is still what a test of the host step would run. + [...moduleIds].map((moduleId) => publishLinkedAgentTools(path, at.ws, scope, moduleId, true)) ) } diff --git a/frontend/src/lib/components/flows/content/AgentResourceBar.svelte b/frontend/src/lib/components/flows/content/AgentResourceBar.svelte index 3a870f911d..96548b10af 100644 --- a/frontend/src/lib/components/flows/content/AgentResourceBar.svelte +++ b/frontend/src/lib/components/flows/content/AgentResourceBar.svelte @@ -4,7 +4,7 @@ import Badge from '$lib/components/common/badge/Badge.svelte' import Path from '$lib/components/Path.svelte' import TextInput from '$lib/components/text_input/TextInput.svelte' - import { ResourceService, type InputTransform } from '$lib/gen' + import { ResourceService, type InputTransform, type Resource } from '$lib/gen' import { workspaceStore } from '$lib/stores' import { sendUserToast } from '$lib/toast' import { Bot, ChevronDown, ChevronUp, Save, Unlink, Pencil } from 'lucide-svelte' @@ -19,14 +19,24 @@ type AIAgentConfig, type AgentTool } from '../agentResourceUtils' - import { agentWriteCount, markAgentWritten, openAgentEditor } from '../agentEditorStore.svelte' + import { + agentDraftSaveCount, + agentWriteCount, + markAgentWritten, + openAgentEditor + } from '../agentEditorStore.svelte' import { setLinkedAgentTools, clearLinkedAgentTools, + linkedModulesForAgent, linkedToolsScope } from '../linkedAgentToolsStore.svelte' import { logReusableAgentUsage } from '../agentTelemetry' import { claimLinkedToolsFetch } from '../flowState' + import { AgentDraftUnavailable, fetchAgentWithDraft } from '../linkedAgentDrafts' + import type { AgentResourceState } from '../agentDraft.svelte' + import { getLocalDraftHint } from '$lib/localDraftHints.svelte' + import Tooltip from '$lib/components/meltComponents/Tooltip.svelte' import type { AgentTool as AgentToolStrict } from '../agentToolUtils' import { resource } from 'runed' import { untrack } from 'svelte' @@ -65,6 +75,10 @@ // deploy from the agent editor mounted alongside it. Both reads below key on it, so neither // keeps naming the config and version a write has just replaced. let writes = $derived(agentWriteCount(ws, agent)) + // Draft saves as well, for the link fetch: the card shows what a test of this step would run, + // and that is the draft. Only the deploy moves `writes`, so without this the card would keep + // describing the config the agent held before it was edited. + let draftSaves = $derived(agentDraftSaveCount(ws, agent)) let saveDrawer: Drawer | undefined = $state() let newPath = $state('') @@ -75,14 +89,18 @@ type LinkedInfo = { // What this result was fetched for. runed's resource neither aborts nor tags a superseded // request, so a slow fetch can land after a newer one: every consumer gates on these matching - // the current (ws, agent, writes). `writes` is what covers a refetch of the *same* link after - // a deploy — without it a pre-deploy response is indistinguishable from the current one, and - // accepting it republishes the tools the deploy just replaced. + // the current (ws, agent, writes, draftSaves). `writes` is what covers a refetch of the *same* + // link after a deploy — without it a pre-deploy response is indistinguishable from the current + // one, and accepting it republishes the tools the deploy just replaced. `draftSaves` does the + // same for a draft save, which the card follows just as closely. ws?: string path?: string writes: number + draftSaves: number config: AIAgentConfig tools: AgentTool[] + /** The config shown came from the agent's unsaved draft rather than the deployed resource. */ + fromDraft: boolean providerPath?: string providerOk: boolean } @@ -90,14 +108,37 @@ // A linked agent is rigid and read-only: its brain and tools come from the resource. We // load them here for display, and probe the provider resource so we can warn when it isn't // accessible in this workspace (the user then needs to unlink/fork or gain access). + // The draft when there is one, since that is what a test of this step runs. let linkedResource = resource( - () => ({ ws, path: agent, writes }), - async ({ ws, path, writes }): Promise => { + () => ({ ws, path: agent, writes, draftSaves }), + async ({ ws, path, writes, draftSaves }): Promise => { if (!ws || !path) { - return { ws, path, writes, config: {}, tools: [], providerOk: true } + return { + ws, + path, + writes, + draftSaves, + config: {}, + tools: [], + fromDraft: false, + providerOk: true + } + } + let response: Resource + let draft: AgentResourceState | undefined + try { + ;({ response, draft } = await fetchAgentWithDraft(path, ws)) + } catch (err) { + // Only the DRAFT was unreadable. This card is a display, so fall back to the deployed + // agent rather than rendering one with no brain and no tools, which reads as "the agent + // is empty" while the Draft badge still says it has unsaved changes. Same fallback the + // graph's tool nodes take; the paths that run or deploy the draft still refuse. + if (!(err instanceof AgentDraftUnavailable)) throw err + response = await ResourceService.getResource({ workspace: ws, path }) + } + const cfg = (draft?.args ?? response.value ?? {}) as AIAgentConfig & { + provider?: { resource?: string } } - const res = await ResourceService.getResource({ workspace: ws, path }) - const cfg = (res.value ?? {}) as AIAgentConfig & { provider?: { resource?: string } } const tools = (cfg.tools ?? []) as AgentTool[] const providerRef = cfg.provider?.resource const providerPath = @@ -116,8 +157,10 @@ ws, path, writes, + draftSaves, config: cfg, tools, + fromDraft: draft != undefined, providerPath, providerOk } @@ -129,7 +172,13 @@ let loadedInfo = $state(undefined) $effect(() => { const current = linkedResource.current - if (current && current.ws === ws && current.path === agent && current.writes === writes) { + if ( + current && + current.ws === ws && + current.path === agent && + current.writes === writes && + current.draftSaves === draftSaves + ) { loadedInfo = current } }) @@ -140,6 +189,12 @@ let brainParams = $derived(summarizeAgentBrain(linkedInfo?.config)) let providerPath = $derived(linkedInfo?.providerPath) let providerOk = $derived(linkedInfo?.providerOk ?? true) + // The hint flips on the first keystroke in the agent editor, so the badge does not wait for the + // debounced autosave and the refetch behind it; the fetched answer covers a draft written + // elsewhere, which no editor here has published an opinion about. + let hasDraft = $derived( + getLocalDraftHint(ws, 'resource', agent ?? '') ?? linkedInfo?.fromDraft ?? false + ) /** The agent the card is about: the one this step links to, or the one being edited. */ let cardPath = $derived(agent) // The version eval runs are recorded against. The resource does not hold it; its newest history @@ -190,9 +245,18 @@ } const loaded = linkedInfo if (loaded) { - claimLinkedToolsFetch(toolScope, moduleId) - // linkedResource types tools loosely; they are the same resource tools the store holds. - setLinkedAgentTools(toolScope, moduleId, loaded.tools as AgentToolStrict[], agent) + // Every step of this flow linking this agent, not just this one. Tools belong to the agent, + // so the sibling steps show the same set, and only the selected step mounts this card: + // without them a draft saved from here leaves their nodes on what the flow load resolved, + // while a test of those steps runs the draft. Claimed like this card's own publish, so a + // sibling's in-flight fetch cannot land afterwards and put the old tools back. + const modules = new Set(linkedModulesForAgent(toolScope, agent)) + modules.add(moduleId) + for (const id of modules) { + claimLinkedToolsFetch(toolScope, id) + // linkedResource types tools loosely; they are the same resource tools the store holds. + setLinkedAgentTools(toolScope, id, loaded.tools as AgentToolStrict[], agent) + } publishedFor = agent } else if (publishedFor !== undefined && publishedFor !== agent) { // The link moved and the new agent hasn't resolved, so the stored tools are the old one's. @@ -358,13 +422,15 @@ // `tools` is one array per module value, so it identifies the step itself — the path alone // would not, since a replacement can carry the same link. const stepMarker = tools - const res = await ResourceService.getResource({ workspace: ws, path }) + // The draft, like the card above and like a test of this step: forking the deployed value + // while the card displays a drafted prompt would hand back something the user never saw. + const { response, draft } = await fetchAgentWithDraft(path, ws) // The module may have been replaced while the fetch was in flight (undo, session drafts); // applying a stale fork would overwrite the restored state. if (agent !== path || tools !== stepMarker) { return false } - const cfg = (res.value ?? {}) as AIAgentConfig + const cfg = (draft?.args ?? response.value ?? {}) as AIAgentConfig // Preserve the flow-local inputs already wired in the step. const local: Record = {} for (const key of AGENT_FLOW_LOCAL_KEYS) { @@ -448,6 +514,15 @@ v{version} {/if} + {#if hasDraft} + + Draft + {#snippet text()} + This agent has unsaved changes. Testing this flow runs the draft, and deploying the + flow offers to deploy it. + {/snippet} + + {/if}
{#if brainParams.length > 0 || inheritedTools.length > 0} diff --git a/frontend/src/lib/components/flows/flowState.ts b/frontend/src/lib/components/flows/flowState.ts index 3123396a9d..595227f57d 100644 --- a/frontend/src/lib/components/flows/flowState.ts +++ b/frontend/src/lib/components/flows/flowState.ts @@ -5,6 +5,7 @@ import { get } from 'svelte/store' import { workspaceStore } from '$lib/stores' import { isFlowModuleTool, agentToolToFlowModule, type AgentTool } from './agentToolUtils' import { linkedToolsScope, setLinkedAgentTools } from './linkedAgentToolsStore.svelte' +import { fetchAgentWithDraft, normalizeAgentRef } from './linkedAgentDrafts' import { loadFlowModuleState } from './flowStateUtils.svelte' import { emptyFlowModuleState } from './utils.svelte' import type { StateStore } from '$lib/utils' @@ -90,7 +91,9 @@ async function mapFlowModule( // the graph can render its tool nodes. They are display-only (their inputs are edited in // the step panel, which infers schemas itself), so no per-tool module state is loaded — // resource tool ids are not flow-unique and must not key into the flow state. - await publishLinkedAgentTools(agentRef, workspace, scope, flowModule.id) + // Drafts included: every caller of `initFlowState` is a flow editor, where the graph has + // to show the tools a test would run. Read-only viewers publish for themselves. + await publishLinkedAgentTools(agentRef, workspace, scope, flowModule.id, true) } else { // Shape-checked because `tools` is JSON-authored: throwing here would skip the agent's // own state below, leaving it with no schema rather than with no tool schemas. @@ -119,11 +122,17 @@ export async function publishLinkedAgentTools( agentRef: string, workspace: string | undefined, scope: string, - moduleId: string + moduleId: string, + /** Resolve from the agent's unsaved draft when there is one. Editors pass true so the graph + * shows the tool set a test would run; read-only viewers pass false, since a run they are + * displaying used the deployed agent. Required rather than defaulted: an editor call site that + * forgets it republishes the deployed tools over the drafted ones, which reads as the graph + * spontaneously reverting. */ + withDraft: boolean ) { const genKey = `${scope}:${moduleId}` const gen = claimLinkedToolsFetch(scope, moduleId) - const tools = await resolveLinkedAgentTools(agentRef, workspace) + const tools = await resolveLinkedAgentTools(agentRef, workspace, withDraft) if (linkedToolFetchGen.get(genKey) === gen) { setLinkedAgentTools(scope, moduleId, tools, agentRef) } @@ -155,12 +164,27 @@ export function claimLinkedToolsFetch(scope: string, moduleId: string): number { // resource is missing or inaccessible so a broken link never stalls the flow load. export async function resolveLinkedAgentTools( agentRef: string, - workspace?: string + workspace: string | undefined, + withDraft: boolean ): Promise { const ws = workspace ?? get(workspaceStore) if (!ws) return [] - const path = agentRef.replace(/^\$res:/, '').replace(/^res:\/\//, '') + const path = normalizeAgentRef(agentRef) try { + if (withDraft) { + try { + const { response, draft } = await fetchAgentWithDraft(path, ws) + const value = (draft?.args ?? response.value) as { tools?: AgentTool[] } | undefined + return (value?.tools ?? []) as AgentTool[] + } catch { + // The draft read failed for any reason. This is a display, not a run, so fall through to + // the deployed tools rather than showing an agent with none: an empty node list reads as + // "the agent lost its tools" instead of "we could not reach the server". The paths that + // act on a draft — the previews and the deploy dialog — surface the failure instead. + // Not rethrowing anything here: the outer catch turns every throw into `[]`, so a + // rethrow would skip the very fallback this exists for. + } + } const res = await ResourceService.getResource({ workspace: ws, path }) return ((res.value as { tools?: AgentTool[] } | undefined)?.tools ?? []) as AgentTool[] } catch { diff --git a/frontend/src/lib/components/flows/linkedAgentDrafts.test.ts b/frontend/src/lib/components/flows/linkedAgentDrafts.test.ts new file mode 100644 index 0000000000..563007615f --- /dev/null +++ b/frontend/src/lib/components/flows/linkedAgentDrafts.test.ts @@ -0,0 +1,148 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +import { + inlineAgentDraft, + inlineAgentDrafts, + loadLinkedAgentDrafts, + type LinkedAgentDraft +} from './linkedAgentDrafts' +import { ResourceService, type FlowModule, type FlowValue } from '$lib/gen' + +type AiAgentValue = Extract + +function linkedStep(input_transforms: Record): AiAgentValue { + return { + type: 'aiagent', + agent: 'f/team/support', + tool_inputs: { t1: { query: { type: 'javascript', expr: 'flow_input.q' } } }, + input_transforms + } as unknown as AiAgentValue +} + +describe('inlineAgentDraft', () => { + // The overlay order is the worker's: the resource brain first, the step's flow-local inputs on + // top. Reversing it would run the agent author's own `user_message` instead of the flow's. + it('keeps the step wired to the flow while the brain comes from the draft', () => { + const inlined = inlineAgentDraft( + linkedStep({ + user_message: { type: 'javascript', expr: 'flow_input.question' }, + user_attachments: { type: 'static', value: [] } + }), + { + provider: { kind: 'openai', model: 'gpt-4o', resource: '$res:f/team/openai' }, + system_prompt: 'answer in french', + user_message: 'the default the agent carries', + tools: [{ id: 't1', summary: 'search' }] + } as any + ) + + expect(inlined.agent).toBeUndefined() + expect(inlined.tools).toEqual([{ id: 't1', summary: 'search' }]) + expect(inlined.input_transforms).toEqual({ + provider: { + type: 'static', + value: { kind: 'openai', model: 'gpt-4o', resource: '$res:f/team/openai' } + }, + system_prompt: { type: 'static', value: 'answer in french' }, + user_message: { type: 'javascript', expr: 'flow_input.question' }, + user_attachments: { type: 'static', value: [] } + }) + // Host bindings are the step's, not the agent's, and the worker overlays them either way. + expect(inlined.tool_inputs).toEqual({ + t1: { query: { type: 'javascript', expr: 'flow_input.q' } } + }) + }) + + // A linked step carries only the flow-local inputs, but one persisted before linking existed can + // still hold stale brain transforms. They must not shadow the draft the test is meant to run. + it('drops brain transforms the step still carries', () => { + const inlined = inlineAgentDraft( + linkedStep({ + user_message: { type: 'static', value: 'hi' }, + system_prompt: { type: 'static', value: 'stale' } + }), + { system_prompt: 'from the draft' } as any + ) + + expect(inlined.input_transforms).toEqual({ + system_prompt: { type: 'static', value: 'from the draft' }, + user_message: { type: 'static', value: 'hi' } + }) + }) +}) + +describe('inlineAgentDrafts', () => { + // The index is keyed on the bare path while a step may name its agent `$res:`-prefixed, and the + // walk has to reach inside branches and loops. Miss either and every preview silently runs the + // deployed agent — the failure this whole path exists to prevent, and a silent one. + it('reaches a $res:-prefixed link nested in a branch', () => { + const value = { + modules: [ + { + id: 'b', + value: { + type: 'branchone', + default: [], + branches: [ + { + modules: [ + { + id: 'inner', + value: { + type: 'aiagent', + agent: '$res:f/team/support', + tools: [], + input_transforms: { user_message: { type: 'static', value: 'hi' } } + } + } + ] + } + ] + } + } + ] + } as unknown as FlowValue + + const drafts = new Map([ + ['f/team/support', { args: { system_prompt: 'drafted' } } as unknown as LinkedAgentDraft] + ]) + + const inner = (inlineAgentDrafts(value, drafts).modules[0].value as any).branches[0].modules[0] + expect(inner.value.agent).toBeUndefined() + expect(inner.value.input_transforms.system_prompt).toEqual({ + type: 'static', + value: 'drafted' + }) + // The input the flow supplies survives the rewrite. + expect(inner.value.input_transforms.user_message).toEqual({ type: 'static', value: 'hi' }) + }) +}) + +// A link the user cannot resolve is an ordinary state and must not block the flow; anything else is +// an outage, and answering "no draft" to one would silently test or deploy against the deployed +// agent while the editor shows the draft. +describe('loadLinkedAgentDrafts error handling', () => { + function failWith(status: number | undefined) { + return async () => { + const err: Error & { status?: number } = new Error('boom') + err.status = status + throw err + } + } + + beforeEach(() => { + vi.restoreAllMocks() + }) + + it.each([401, 403, 404])('treats %i as no draft', async (status) => { + vi.spyOn(ResourceService, 'getResource').mockImplementation(failWith(status) as any) + await expect(loadLinkedAgentDrafts(['f/team/support'], 'ws')).resolves.toEqual(new Map()) + }) + + it.each([500, undefined])('propagates %s rather than reporting no draft', async (status) => { + vi.spyOn(ResourceService, 'getResource').mockImplementation(failWith(status) as any) + await expect(loadLinkedAgentDrafts(['f/team/support'], 'ws')).rejects.toThrow( + 'Could not load the agent f/team/support' + ) + }) +}) diff --git a/frontend/src/lib/components/flows/linkedAgentDrafts.ts b/frontend/src/lib/components/flows/linkedAgentDrafts.ts new file mode 100644 index 0000000000..8da732e7ff --- /dev/null +++ b/frontend/src/lib/components/flows/linkedAgentDrafts.ts @@ -0,0 +1,229 @@ +import { + ResourceService, + type FlowModule, + type FlowValue, + type InputTransform, + type Resource +} from '$lib/gen' +import { UserDraft } from '$lib/userDraft.svelte' +import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte' +import { canWrite } from '$lib/utils' +import type { UserExt } from '$lib/stores' +import { dfs } from './dfs' +import { flowLocalInputs, type AIAgentConfig } from './agentResourceUtils' +import type { AgentResourceState } from './agentDraft.svelte' +import type { AgentTool } from './agentToolUtils' + +/** A step names its agent bare or as `$res:`/`res://`; all three are the same agent, + * and a draft index has to answer for a lookup written any of those ways. Same normalization as + * `linkedAgentToolsStore`, and as the `trim_start_matches` the worker applies. */ +export function normalizeAgentRef(agentRef: string): string { + return agentRef.replace(/^\$res:/, '').replace(/^res:\/\//, '') +} + +/** Every `ai_agent` resource this flow links to, deduped. `dfs` walks agent tool nodes as well as + * branches and loops, so a nested linked agent tool is included. */ +export function linkedAgentPaths(value: FlowValue | undefined): string[] { + if (!value?.modules) return [] + const paths = new Set() + for (const module of dfs(value.modules, (m) => m)) { + const v = module?.value as { type?: string; agent?: string } | undefined + if (v?.type === 'aiagent' && v.agent) { + paths.add(normalizeAgentRef(v.agent)) + } + } + return [...paths] +} + +/** + * The unsaved draft for an agent, freshest first: the cell an open agent editor is writing, then + * what a `get_draft` response carried. + * + * Only the live cell is reliably current, and only while an editor holds it: `releaseEntry` drops + * the cached write at refcount 0 on purpose, so once the agent editor closes the persisted row is + * the sole answer. Read through `fetchAgentWithDraft`, which settles that row first. + */ +export function agentDraftState( + response: { draft?: unknown }, + path: string, + workspace: string | undefined +): AgentResourceState | undefined { + const live = UserDraft.get('resource', path, { workspace }) + return live ?? (response.draft as AgentResourceState | undefined) +} + +/** A refusal that already names the agent and says what is wrong with it, so a caller wrapping it + * would only repeat itself. */ +export class AgentDraftUnavailable extends Error {} + +/** + * An agent's resource together with the draft a run of it would use. + * + * The flush is what makes the answer current. Autosave is debounced by 1.5s (10s ceiling), and + * closing the agent editor releases the in-memory cell without cancelling that pending POST — so + * testing or deploying right after closing would otherwise read a row the last edits have not + * reached yet. `flush` replays the parked save and is a no-op when there is none. + */ +export async function fetchAgentWithDraft( + path: string, + workspace: string +): Promise<{ response: Resource; draft: AgentResourceState | undefined }> { + const query = { workspace, itemKind: 'resource' as const, path } + await UserDraftDbSyncer.flush(query) + // `flush` resolves whether or not the save actually landed: `postSave` catches network and + // server errors into its failure map, and answers a conflicting write by parking a snapshot, + // returning normally in both cases. The row about to be read is then older than the edit still + // held in the browser, and nothing downstream could tell. Running that row is a test of the + // wrong agent; deploying it is worse, because the deploy deletes the draft and takes the newer + // edit with it. Neither is recoverable from here, so refuse the read. + const failure = UserDraftDbSyncer.getState(query).failureMessage + if (failure) { + throw new AgentDraftUnavailable(`The unsaved changes to ${path} could not be saved: ${failure}`) + } + if (UserDraftDbSyncer.getConflict(query).conflict) { + throw new AgentDraftUnavailable( + `The unsaved changes to ${path} could not be saved because it was edited elsewhere. Open the agent to resolve it.` + ) + } + const response = await ResourceService.getResource({ workspace, path, getDraft: true }) + return { response, draft: agentDraftState(response, path, workspace) } +} + +/** One linked agent whose resource the user has an unsaved draft for. */ +export interface LinkedAgentDraft { + path: string + /** The draft's resource value: what a run of this agent would use. */ + args: AIAgentConfig + /** The whole draft row, as the resource editors write it — the deploy payload. */ + state: AgentResourceState + /** No deployed row at this path, so deploying has to create rather than update. */ + noDeployed: boolean + /** Of the deployed resource, for `agentDraftCanWrite`. */ + extraPerms: Record +} + +/** Whether `user` may write this agent's resource. Split from the load so that resolving the + * drafts of a whole flow costs no `whoami` — only the deploy dialog needs the answer, and it + * looks the user up once for every agent it lists. */ +export function agentDraftCanWrite(draft: LinkedAgentDraft, user: UserExt | undefined): boolean { + return canWrite(draft.path, draft.extraPerms, user) +} + +/** A link that cannot resolve for the user rather than because something went wrong: the agent was + * deleted, or sits in a folder they cannot read. Both are ordinary states of a rigid link, and + * neither should stop the caller — the flow still tests and deploys, against the deployed agent. + * Every other failure is an outage, and answering "no draft" to one would quietly run or deploy + * the wrong configuration, which is the whole thing this module exists to prevent. */ +function isExpectedLinkFailure(err: unknown): boolean { + const status = (err as { status?: number } | null | undefined)?.status + return status === 401 || status === 403 || status === 404 +} + +/** + * The unsaved draft of every given `ai_agent` path, for the paths that have one. + * + * Throws when a path fails to load for any reason other than being missing or unreadable, so a + * caller cannot mistake an outage for an agent with nothing unsaved. + */ +export async function loadLinkedAgentDrafts( + paths: string[], + workspace: string | undefined +): Promise> { + const out = new Map() + if (!workspace || paths.length === 0) return out + await Promise.all( + paths.map(async (path) => { + let response: Resource + let draft: AgentResourceState | undefined + try { + ;({ response, draft } = await fetchAgentWithDraft(path, workspace)) + } catch (err) { + if (isExpectedLinkFailure(err)) return + if (err instanceof AgentDraftUnavailable) throw err + throw new Error(`Could not load the agent ${path}: ${err}`) + } + if (!draft) return + out.set(path, { + path, + args: (draft.args ?? {}) as AIAgentConfig, + state: draft, + noDeployed: Boolean((response as { no_deployed?: boolean }).no_deployed), + extraPerms: response.extra_perms ?? {} + }) + }) + ) + return out +} + +/** + * Every argument a saved agent carries, as a static input transform. Not only the keys the agent + * form renders: a run reads them all, and an agent holding its own `user_message` answers with it + * when nothing overrides it. `tools` is the step's own roster rather than an input, so it rides on + * the module's `tools` key instead. + */ +export function agentArgsToTransforms(args: AIAgentConfig): Record { + const it: Record = {} + for (const [key, value] of Object.entries(args ?? {})) { + if (key === 'tools' || value === undefined) continue + it[key] = { type: 'static', value } as InputTransform + } + return it +} + +type AiAgentValue = Extract + +/** + * The standalone step a linked step's draft would run as: the draft's brain and tools inlined, with + * the step's own flow-local inputs kept on top. + * + * The overlay order is the worker's (`ai_executor.rs`): its linked branch interpolates the whole + * resource brain and only then writes `user_message`/`user_attachments` back from the step's own + * args. `tool_inputs` stays untouched — the worker overlays it onto the tools in both branches, so + * an inlined step keeps the host flow's tool bindings. + */ +export function inlineAgentDraft(value: AiAgentValue, args: AIAgentConfig): AiAgentValue { + const { agent: _agent, ...rest } = value + return { + ...rest, + tools: (args.tools ?? []) as AgentTool[], + input_transforms: { + ...agentArgsToTransforms(args), + ...flowLocalInputs(value.input_transforms as Record) + } + } as AiAgentValue +} + +/** + * Replace every linked agent step that has a draft with the draft's own configuration, so a preview + * runs what the agent editor is showing rather than the deployed resource. Returns a new value: the + * flow editor hands its live store object to previews. + */ +export function inlineAgentDrafts( + value: FlowValue, + drafts: Map +): FlowValue { + if (drafts.size === 0) return value + // JSON rather than `structuredClone`: the flow editor's value is a Svelte `$state` proxy, which + // `structuredClone` refuses outright. A flow value is JSON by definition — it is about to be + // posted as one — so the round trip loses nothing this preview would have carried. + const next = JSON.parse(JSON.stringify(value)) as FlowValue + for (const module of dfs(next.modules ?? [], (m) => m)) { + const v = module?.value as AiAgentValue | undefined + if (v?.type !== 'aiagent' || !v.agent) continue + const draft = drafts.get(normalizeAgentRef(v.agent)) + if (!draft) continue + module.value = inlineAgentDraft(v, draft.args) + } + return next +} + +/** Load the drafts this flow's linked agents have and inline them. The whole substitution, for a + * caller holding nothing but the value it is about to preview. */ +export async function withAgentDrafts( + value: FlowValue, + workspace: string | undefined +): Promise { + const paths = linkedAgentPaths(value) + if (paths.length === 0) return value + return inlineAgentDrafts(value, await loadLinkedAgentDrafts(paths, workspace)) +} diff --git a/frontend/src/lib/components/flows/linkedToolsFetchGuard.test.ts b/frontend/src/lib/components/flows/linkedToolsFetchGuard.test.ts index ea1674e0b6..29207f58f7 100644 --- a/frontend/src/lib/components/flows/linkedToolsFetchGuard.test.ts +++ b/frontend/src/lib/components/flows/linkedToolsFetchGuard.test.ts @@ -8,7 +8,9 @@ vi.mock('./agentToolUtils', () => ({ isFlowModuleTool: () => false, agentToolToFlowModule: (t: unknown) => t })) -vi.mock('$lib/stores', () => ({ workspaceStore: { subscribe: (f: (v: string) => void) => (f('ws'), () => {}) } })) +vi.mock('$lib/stores', () => ({ + workspaceStore: { subscribe: (f: (v: string) => void) => (f('ws'), () => {}) } +})) import { claimLinkedToolsFetch, @@ -38,8 +40,8 @@ describe('linked tools fetch guard', () => { ) .mockResolvedValueOnce({ value: { tools: [tool('new')] } } as never) - const stale = publishLinkedAgentTools('f/a/old', 'ws', scope, 'step') - await publishLinkedAgentTools('f/a/new', 'ws', scope, 'step') + const stale = publishLinkedAgentTools('f/a/old', 'ws', scope, 'step', false) + await publishLinkedAgentTools('f/a/new', 'ws', scope, 'step', false) release?.({ value: { tools: [tool('old')] } }) await stale @@ -56,7 +58,7 @@ describe('linked tools fetch guard', () => { () => new Promise((r) => (release = r)) as ReturnType ) - const inFlight = publishLinkedAgentTools('f/a/old', 'ws', scope, 'step') + const inFlight = publishLinkedAgentTools('f/a/old', 'ws', scope, 'step', false) setLinkedAgentTools(scope, 'step', [tool('kept')], 'u/admin/a') invalidateLinkedToolsFetches(scope) @@ -73,7 +75,7 @@ describe('linked tools fetch guard', () => { () => new Promise((r) => (release = r)) as ReturnType ) - const inFlight = publishLinkedAgentTools('f/a/old', 'ws', scope, 'step') + const inFlight = publishLinkedAgentTools('f/a/old', 'ws', scope, 'step', false) claimLinkedToolsFetch(scope, 'step') setLinkedAgentTools(scope, 'step', [tool('direct')], 'u/admin/a') release?.({ value: { tools: [tool('stale')] } }) diff --git a/frontend/src/lib/components/flows/utils.svelte.ts b/frontend/src/lib/components/flows/utils.svelte.ts index 8d8e4366e5..df27960dc7 100644 --- a/frontend/src/lib/components/flows/utils.svelte.ts +++ b/frontend/src/lib/components/flows/utils.svelte.ts @@ -17,6 +17,7 @@ import { get } from 'svelte/store' import type { FlowModuleState } from './flowState' import { type PickableProperties, dfs } from './previousResults' import { forEachFlowModule } from './dfs' +import { withAgentDrafts } from './linkedAgentDrafts' import { NEVER_TESTED_THIS_FAR } from './models' import { sendUserToast } from '$lib/toast' import type { ExtendedOpenFlow } from './types' @@ -187,6 +188,12 @@ export function jobsToResults(jobs: Job[]) { }) } +/** + * Run the flow the editor currently holds. A step linked to a saved agent runs that agent's + * unsaved draft when there is one (`withAgentDrafts`), so testing exercises what the agent editor + * is showing rather than the deployed resource — the same rule the agent editor's own test pane + * follows. The value passed in is left alone; only what goes to the server is substituted. + */ export async function runFlowPreview( args: Record, flow: OpenFlow & { tag?: string }, @@ -198,14 +205,15 @@ export async function runFlowPreview( // editor; falls back to the navigation workspace for full-page previews. workspace?: string ) { - const newFlow = flow + const ws = workspace ?? get(workspaceStore) ?? '' + const value = await withAgentDrafts(flow.value, ws) return await JobService.runFlowPreview({ - workspace: workspace ?? get(workspaceStore) ?? '', + workspace: ws, requestBody: { args, - value: newFlow.value, + value, path: path, - tag: newFlow.tag, + tag: flow.tag, restarted_from: restartedFrom, temp_script_refs: tempScriptRefs }, diff --git a/frontend/src/lib/userDraft.svelte.ts b/frontend/src/lib/userDraft.svelte.ts index 24572ee1ab..7942694f31 100644 --- a/frontend/src/lib/userDraft.svelte.ts +++ b/frontend/src/lib/userDraft.svelte.ts @@ -349,6 +349,32 @@ export const UserDraft = { void UserDraftDbSyncer.save({ workspace: ws, itemKind, path, value: null }) }, + /** + * Drop this key's in-memory cell and cached write WITHOUT touching the + * server. For callers that have already deleted the row by another route + * (a deploy) and only need the local mirror to stop answering `get`/`has` + * with a value that is gone. + * + * MUST be used instead of `remove` there. `remove` POSTs its own + * `value: null`, and that POST is debounced and carries whatever + * `last_sync` is left — which a preceding successful delete has already + * cleared. The backend treats a delete with no `last_sync` as + * unconditional, so the second POST lands ~1.5s later with nothing to + * compare against and removes a draft saved in the meantime. + */ + forgetLocal(itemKind: UserDraftItemKind, path: string, opts?: UserDraftOptions): void { + const ws = resolveWorkspace(opts) + const mk = mapKey(ws, itemKind, path) + const entry = entries.get(mk) + if (entry) { + // Same as `remove`: clear the cell so live observers see the delete, and arm + // `skipNextSync` so the mirror does not turn that write into a POST of its own. + entry.skipNextSync = true + entry.state.val = undefined + } + writtenCache.delete(mk) + }, + clear(itemKind: UserDraftItemKind, path: string, opts?: UserDraftOptions): void { UserDraft.discard(itemKind, path, undefined, opts) }, diff --git a/frontend/src/lib/utils_draft_deploy.test.ts b/frontend/src/lib/utils_draft_deploy.test.ts index 9bed95917b..4495155f24 100644 --- a/frontend/src/lib/utils_draft_deploy.test.ts +++ b/frontend/src/lib/utils_draft_deploy.test.ts @@ -7,7 +7,7 @@ vi.mock('$lib/gen', () => ({ DraftService: { deleteDraft: vi.fn() }, AppService: {}, VariableService: {}, - ResourceService: {}, + ResourceService: { getResource: vi.fn(), updateResource: vi.fn(), createResource: vi.fn() }, ScheduleService: {}, HttpTriggerService: {}, WebsocketTriggerService: {}, @@ -21,7 +21,9 @@ vi.mock('$lib/gen', () => ({ AzureTriggerService: {}, EmailTriggerService: {} })) -vi.mock('$lib/userDraftDbSyncer.svelte', () => ({ UserDraftDbSyncer: { save: vi.fn() } })) +vi.mock('$lib/userDraftDbSyncer.svelte', () => ({ + UserDraftDbSyncer: { save: vi.fn(), recordRemoteSync: vi.fn() } +})) vi.mock('$lib/workspaceDrafts.svelte', () => ({ invalidateWorkspaceDrafts: vi.fn() })) vi.mock('$lib/workspaceComparison', () => ({ invalidateWorkspaceComparison: vi.fn() })) vi.mock('$lib/localDraftHints.svelte', () => ({ setLocalDraftHint: vi.fn() })) @@ -30,7 +32,8 @@ vi.mock('$lib/components/raw_apps/utils', () => ({ canonicalRawAppDiffValue: vi. vi.mock('$lib/appDiffSides', () => ({ classicAppDraftParts: vi.fn() })) vi.mock('$lib/utils_deployable', () => ({ TRIGGER_RUNTIME_IGNORE: [] })) -import { ScriptService, FlowService } from '$lib/gen' +import { ScriptService, FlowService, ResourceService } from '$lib/gen' +import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte' // draftBaseIsStale compares a draft's base pointer against the deployed head // of the item it was fetched with (`get_draft=true`). Shared by CompareDrafts @@ -124,3 +127,65 @@ describe('deployDraft preserves on_behalf_of', () => { ) }) }) + +// The resource branch reads the item again when the deploy lands, and falls back to the deployed +// row when the draft has gone. That row keeps its value under `value` and carries no `args` at +// all, so reading it as a draft (`value: d.args ?? {}`) would replace a live resource with `{}`. +describe('deployDraft: resource with no draft', () => { + beforeEach(() => vi.clearAllMocks()) + + it('writes nothing rather than `{}` over the deployed value', async () => { + vi.mocked(ResourceService.getResource).mockResolvedValueOnce({ + path: 'f/support/triage_agent', + resource_type: 'ai_agent', + value: { system_prompt: 'deployed' } + } as any) + + // `noop` is what lets a caller deploying one specific draft tell "nothing to promote" apart + // from "deployed", instead of reporting an agent as deployed that was never written. + expect(await deployDraft('resource', 'f/support/triage_agent', 'ws')).toEqual({ + success: true, + noop: true + }) + expect(ResourceService.updateResource).not.toHaveBeenCalled() + expect(ResourceService.createResource).not.toHaveBeenCalled() + // Nor does it touch the draft row. There was none of this user's to delete, so the only row + // the cleanup could reach is one written after the read: an edit destroyed without ever + // having been deployed. Clearing the baseline would be the same bug by another route, since + // a delete with no baseline is the unconditional one. + expect(UserDraftDbSyncer.save).not.toHaveBeenCalled() + expect(UserDraftDbSyncer.recordRemoteSync).not.toHaveBeenCalled() + }) + + it('still deploys normally when the draft is there, and keys the cleanup to the row it read', async () => { + vi.mocked(ResourceService.getResource).mockResolvedValueOnce({ + path: 'f/support/triage_agent', + resource_type: 'ai_agent', + value: { system_prompt: 'deployed' }, + draft_saved_at: '2026-01-01T00:00:00Z', + draft: { path: 'f/support/triage_agent', args: { system_prompt: 'drafted' } } + } as any) + + expect(await deployDraft('resource', 'f/support/triage_agent', 'ws')).toEqual({ success: true }) + expect(ResourceService.updateResource).toHaveBeenCalledWith( + expect.objectContaining({ + requestBody: expect.objectContaining({ value: { system_prompt: 'drafted' } }) + }) + ) + // The draft delete that follows is conditional on this baseline. With no baseline the backend + // deletes unconditionally, destroying a draft saved between the read and the delete without + // ever having deployed it, so the timestamp has to be the one from the row just promoted. + expect(UserDraftDbSyncer.recordRemoteSync).toHaveBeenCalledWith( + { workspace: 'ws', itemKind: 'resource', path: 'f/support/triage_agent' }, + '2026-01-01T00:00:00Z' + ) + expect(UserDraftDbSyncer.save).toHaveBeenCalledWith( + expect.objectContaining({ path: 'f/support/triage_agent', value: null, immediate: true }) + ) + // Order is the whole point: a delete issued before the seed carries whatever baseline the tab + // happened to hold, which for a caller that only read through a listing is none at all. + expect(vi.mocked(UserDraftDbSyncer.recordRemoteSync).mock.invocationCallOrder[0]).toBeLessThan( + vi.mocked(UserDraftDbSyncer.save).mock.invocationCallOrder[0] + ) + }) +}) diff --git a/frontend/src/lib/utils_draft_deploy.ts b/frontend/src/lib/utils_draft_deploy.ts index b322d6adc4..f322bb27b9 100644 --- a/frontend/src/lib/utils_draft_deploy.ts +++ b/frontend/src/lib/utils_draft_deploy.ts @@ -435,8 +435,13 @@ export async function deployDraft( path: string, workspace: string, opts: { draftOnly?: boolean; rawApp?: boolean; deploymentMessage?: string } = {} -): Promise { +): Promise { const { draftOnly = false, rawApp = false, deploymentMessage } = opts + // Set when the branch found nothing to promote and wrote nothing. Success, because the item is + // already at the value a deploy would have left it at and its stale draft state still wants + // clearing — but a caller deploying one specific draft it showed the user has to be able to tell + // that apart from having deployed it. + let noop = false try { if (kind === 'raw_app' || (kind === 'app' && rawApp)) { // Raw apps bundle their source files and deploy via the raw-app @@ -579,10 +584,34 @@ export async function deployDraft( } void deployed } else if (kind === 'resource') { - const { deployed, draft: d } = splitOverlay(await OVERLAY_GETTERS.resource!(workspace, path)) + const overlay = await OVERLAY_GETTERS.resource!(workspace, path) + // Adopt the row this promote is based on as the baseline for the delete below. Without one + // the backend deletes unconditionally, so a draft saved between that read and the delete is + // destroyed having never been deployed — a caller that only ever read through a listing has + // no baseline of its own to supply. With it the delete is refused instead and the newer + // draft survives, which is the recoverable outcome of the two. Only ever seeded, never + // cleared: passing no timestamp drops whatever baseline the tab already held, which would + // turn that same delete back into an unconditional one. + if (overlay?.draft_saved_at) { + UserDraftDbSyncer.recordRemoteSync( + { workspace, itemKind: kind, path }, + overlay.draft_saved_at + ) + } + const { deployed, draft: d, hasDraft } = splitOverlay(overlay) // ResourceEditor's `ResourceState` draft shape: // { path, description, args, resource_type?, labels?, wsSpecific } - if (draftOnly) { + // The deployed row is a different shape (`value`, `ws_specific`, no `args` at all), and + // `splitOverlay` hands it back as the draft side when the draft row has gone — deployed or + // discarded from another tab between the listing and this click. Reading it as a draft is + // what made `value: d.args ?? {}` replace a live resource with `{}`. Nothing to promote + // then, so write nothing and fall through to the cleanup below, which clears the stale + // local draft hint and the drafts listing. The item is already at the value a successful + // deploy would have left it at, so this reports success rather than an error, matching + // what the other kinds end up doing when their own draft is gone. + if (!hasDraft) { + noop = true + } else if (draftOnly) { await ResourceService.createResource({ workspace, requestBody: { @@ -633,8 +662,8 @@ export async function deployDraft( return { success: false, error: `Deploy not supported for draft kind ${kind}` } } // Delete the draft at its STORAGE path (the row key, = the `path` arg). - // Two reasons it must happen here for every kind, mirroring the editors' - // post-deploy `discardDraftAfterDeploy(draftPath)`: + // Two reasons it must happen here for every kind that promoted something, + // mirroring the editors' post-deploy `discardDraftAfterDeploy(draftPath)`: // - Drawer kinds (variable / resource / triggers) aren't deleted by // their create/update endpoints at all. // - script/flow/app/raw_app DO delete server-side, but only the draft at @@ -642,13 +671,18 @@ export async function deployDraft( // synthetic `u/{user}/draft_{uuid}` storage path ≠ `d.path`, so its // draft row survives the deploy and keeps listing. Deleting the // storage-path draft removes it (a no-op when the server already did). - await UserDraftDbSyncer.save({ - workspace, - itemKind: kind, - path, - value: null, - immediate: true - }) + // Skipped when nothing was promoted: the read that set `noop` found no draft of this user's to + // delete, so the only row this could reach is one written after it — destroying an edit that + // was never deployed, and never even listed. + if (!noop) { + await UserDraftDbSyncer.save({ + workspace, + itemKind: kind, + path, + value: null, + immediate: true + }) + } // Mutated the workspace's Server Drafts — refresh every mounted reader. invalidateWorkspaceDrafts(workspace) // The DEPLOYED state moved: cached fork comparisons involving this @@ -659,7 +693,7 @@ export async function deployDraft( // so the syncer-owned hint won't auto-clear — clear it explicitly. // (Idempotent: the drawer-kind delete above already cleared it.) setLocalDraftHint(workspace, kind, path, false) - return { success: true } + return noop ? { success: true, noop: true } : { success: true } } catch (e: any) { return { success: false, error: e?.body ?? e?.message ?? String(e) } } From c3f7f8a45830fb548aa628ebf6e2b6c95c6de67f Mon Sep 17 00:00:00 2001 From: Diego Imbert <70353967+diegoimbert@users.noreply.github.com> Date: Mon, 7 Sep 2026 19:31:59 +0200 Subject: [PATCH 14/15] fix: stop an untouched item's form from saving a draft nobody wrote (#10964) * feat: gate drafts on real user input so a moved-on schema is not a draft Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01ScVgqGpuyDMWdzVNPm7Q5f * Revert "feat: gate drafts on real user input so a moved-on schema is not a draft" This reverts commit 6cd86cf727dc46d9836440ae345ca6161c3fb31c. * fix: stop counting empty schema-added fields and server metadata as drafts Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01ScVgqGpuyDMWdzVNPm7Q5f * feat: sweep away existing drafts that carry no changes, once per workspace Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01ScVgqGpuyDMWdzVNPm7Q5f * Reapply "feat: gate drafts on real user input so a moved-on schema is not a draft" This reverts commit b7b18e345e6984544290009247faca1e341b46bd. * Revert "fix: stop counting empty schema-added fields and server metadata as drafts" This reverts commit 9787270ad883722735d3873ee98d37a7fd11ad23. * docs: describe the sweep by the gate that now prevents new phantom drafts Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01ScVgqGpuyDMWdzVNPm7Q5f * fix: make the draft sweep a compare-and-delete so it cannot eat live edits Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01ScVgqGpuyDMWdzVNPm7Q5f * fix: close the gate's load-time window and stop sealing a failed sweep Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01ScVgqGpuyDMWdzVNPm7Q5f * fix: open the gate on the edit itself, and stop the sweep at ownerless drafts Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01ScVgqGpuyDMWdzVNPm7Q5f * fix: count a click as an edit, and keep an unjudged row from sealing the sweep Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01ScVgqGpuyDMWdzVNPm7Q5f * fix: release the sweep's sync baseline when its delete is refused Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01ScVgqGpuyDMWdzVNPm7Q5f * fix: drop the refused delete before re-baselining, and bound the sweep's retries Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01ScVgqGpuyDMWdzVNPm7Q5f * refactor: send the sweep's delete straight to the API, not through the syncer Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01ScVgqGpuyDMWdzVNPm7Q5f * fix: never absorb a change the resource type's schema could not have made Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01ScVgqGpuyDMWdzVNPm7Q5f * fix: push an edit the gate only notices after the write has landed Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01ScVgqGpuyDMWdzVNPm7Q5f * fix: stop the gating effect re-suspending a resource opened on a draft Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01ScVgqGpuyDMWdzVNPm7Q5f * chore: update ee-repo-ref to d33ea730c550cdbc7d050aeb6d40dcef3d134e07 This commit updates the EE repository reference after PR #782 was merged in windmill-ee-private. Previous ee-repo-ref: 313c572c9dcbcaafd8a1594df4054f9dd26f395c New ee-repo-ref: d33ea730c550cdbc7d050aeb6d40dcef3d134e07 Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: windmill-internal-app[bot] --- .../src/lib/components/ResourceEditor.svelte | 108 +++++++- .../triggers/useTriggerDraftSync.svelte.ts | 106 ++++++-- frontend/src/lib/userDraft.svelte.ts | 5 +- frontend/src/lib/userDraftEditGate.ts | 66 +++++ frontend/src/lib/userDraftPrune.test.ts | 243 ++++++++++++++++++ frontend/src/lib/userDraftPrune.ts | 230 +++++++++++++++++ frontend/src/lib/utils_draft_deploy.ts | 13 + .../src/routes/(root)/(logged)/+layout.svelte | 11 +- 8 files changed, 759 insertions(+), 23 deletions(-) create mode 100644 frontend/src/lib/userDraftEditGate.ts create mode 100644 frontend/src/lib/userDraftPrune.test.ts create mode 100644 frontend/src/lib/userDraftPrune.ts diff --git a/frontend/src/lib/components/ResourceEditor.svelte b/frontend/src/lib/components/ResourceEditor.svelte index 89acac79de..3c9eda9467 100644 --- a/frontend/src/lib/components/ResourceEditor.svelte +++ b/frontend/src/lib/components/ResourceEditor.svelte @@ -2,7 +2,7 @@ import type { Schema } from '$lib/common' import { ResourceService, WorkspaceService, type Resource, type ResourceType } from '$lib/gen' import { canWrite } from '$lib/utils' - import { createEventDispatcher, untrack } from 'svelte' + import { createEventDispatcher, onDestroy, untrack } from 'svelte' import { userStore, workspaceStore } from '$lib/stores' import { sendUserToast } from '$lib/toast' import { clearJsonSchemaResourceCache } from './schema/jsonSchemaResource.svelte' @@ -13,7 +13,9 @@ import { getUserExt } from '$lib/user' import type { UserExt } from '$lib/stores' import { UserDraft, draftValuesEqual, type UserDraftHandle } from '$lib/userDraft.svelte' + import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte' import { setLocalDraftHint } from '$lib/localDraftHints.svelte' + import { onUserInput } from '$lib/userDraftEditGate' interface Props { canSave?: boolean @@ -108,6 +110,55 @@ workspaceSpecs.push({ ws, defaultValue }) } + // Gated per workspace until the user puts something into that workspace's + // form (see `onUserInput`): the autosave stays suspended and the deployed + // baseline absorbs whatever the form settles on. A workspace opened ON a + // saved draft keeps its baseline — that divergence is the user's own. + let userEdited: Record = $state({}) + let openedOnDraft: Record = $state({}) + const suspendedWorkspaces = new Set() + + function setGated(ws: string, gated: boolean): void { + if (!initialPath) return + if (gated === suspendedWorkspaces.has(ws)) return + if (gated) { + UserDraft.stopSync('resource', initialPath, { workspace: ws }) + suspendedWorkspaces.add(ws) + } else { + UserDraft.restartSync('resource', initialPath, { workspace: ws }) + suspendedWorkspaces.delete(ws) + } + } + + // Nothing counts until this workspace's form is on screen, and while the + // schema is still arriving a precursor alone does not: it would open the gate + // just in time for the schema's materialized values to POST. `Path`, the + // labels and the description render above that skeleton and stay editable + // throughout, so a real value event still counts and keeps the edit. + onUserInput((kind) => { + if (!selected || !(selected in states)) return + if (kind === 'precursor' && loadingSchema) return + userEdited[selected] = true + }) + + $effect(() => { + const wss = Object.keys(states) + const edited = { ...userEdited } + const onDraft = { ...openedOnDraft } + untrack(() => { + // A workspace opened on a saved draft is never suspended — there is no + // phantom to prevent, and a write made while suspended is dropped for + // good. Without `onDraft` here this effect re-suspends it the moment its + // handle appears, undoing the decision made when it was opened. + for (const ws of wss) setGated(ws, !edited[ws] && !onDraft[ws]) + }) + }) + + // `stopSync` must be paired or the key stays unsynced for the session. + onDestroy(() => { + for (const ws of [...suspendedWorkspaces]) setGated(ws, false) + }) + let isValid = $state(true) let jsonError = $state('') let perWsValid: Record = $state({}) @@ -245,6 +296,13 @@ } // Open with the saved draft if present, else the deployed. const s: ResourceState = savedDraftState ?? deployedState + openedOnDraft[ws] = !!savedDraftState + // Gate BEFORE the handle is acquired: `stopSync` queues on a + // not-yet-live entry, and the form can settle before the effect + // above gets a chance to run. Only worth doing when no draft exists + // yet — where one does, there is no phantom to prevent and + // suspending could only drop a write. + if (!savedDraftState) setGated(ws, true) ensureHandle(ws, s) initialStates[ws] = structuredClone(deployedState) // Draft-only paths (`no_deployed`) have no row — saving must @@ -259,6 +317,47 @@ }) }) + /** The schema can only ever write `args`. `path`, `labels`, `description` and + * `wsSpecific` are beyond its reach, so a difference in one of those is the + * user's — whatever event did or didn't reach the gate. Removing a label runs + * a click handler and emits nothing native, and would otherwise be absorbed. */ + function differsOutsideArgs(a: ResourceState, b: ResourceState | undefined): boolean { + return !!b && !draftValuesEqual({ ...a, args: null }, { ...b, args: null }) + } + + // Absorb the form's settling writes into the deployed baseline while the + // selected workspace is gated, so they show up neither as the "unsaved + // changes" banner nor, once `discardIf` reads the baseline, as a draft. + // Only the selected workspace has a form rendered against it. + $effect(() => { + const ws = selected + if (!ws || !initialPath) return + if (userEdited[ws] || openedOnDraft[ws]) return + // `$state.snapshot` deep-reads, so nested `args` mutations re-run this. + const settled = states[ws]?.draft + ? ($state.snapshot(states[ws].draft) as ResourceState) + : undefined + untrack(() => { + if (!settled) return + if (differsOutsideArgs(settled, initialStates[ws])) { + // An edit, not settling. This runs AFTER the write landed, and a write + // made while suspended is swallowed for good (the mirror advances its + // baseline either way), so un-suspend and push the value here rather + // than leaving it to whichever effect happens to run next. + userEdited[ws] = true + setGated(ws, false) + void UserDraftDbSyncer.save({ + workspace: ws, + itemKind: 'resource', + path: initialPath, + value: settled + }) + return + } + if (!draftValuesEqual(settled, initialStates[ws])) initialStates[ws] = settled + }) + }) + // Keep current.path bound to the outer `path` prop for consumers $effect(() => { if (current) path = current.path @@ -292,6 +391,13 @@ } export function discardLocalDraft(): void { if (!selected) return + // Back to the deployed value with nothing of the user's left in it, so + // the gate closes again — otherwise the form settles on the schema's + // values a second time and the discarded draft comes straight back. + // `discard` POSTs the delete itself, so suspending first is safe. + openedOnDraft[selected] = false + userEdited[selected] = false + setGated(selected, true) UserDraft.discard('resource', initialPath ?? '', initialStates[selected], { workspace: selected }) diff --git a/frontend/src/lib/components/triggers/useTriggerDraftSync.svelte.ts b/frontend/src/lib/components/triggers/useTriggerDraftSync.svelte.ts index ba09167d6d..98d38ac25f 100644 --- a/frontend/src/lib/components/triggers/useTriggerDraftSync.svelte.ts +++ b/frontend/src/lib/components/triggers/useTriggerDraftSync.svelte.ts @@ -2,9 +2,21 @@ import { untrack } from 'svelte' import { deepEqual } from 'fast-equals' import { UserDraft, normalizeDraftForCompare, type UserDraftItemKind } from '$lib/userDraft.svelte' import { setLocalDraftHint } from '$lib/localDraftHints.svelte' +import { onUserInput } from '$lib/userDraftEditGate' type Cfg = Record +/** + * Detach a config from whatever holds it. The draft cell is deeply reactive, + * so handing its object straight to `applyCfg` would make the form's own + * `$state` (a schedule's `args`, say) the very object the cell holds — every + * later keystroke would then mutate the draft in place behind the autosave's + * back. + */ +function snapshotCfg(cfg: V): V { + return structuredClone($state.snapshot(cfg)) as V +} + /** * Whether `a` differs from `b` after `normalizeDraftForCompare` (JSON * round-trip to drop `undefined`-valued keys, plus ignored deploy-directive @@ -84,8 +96,10 @@ export interface TriggerDraftSync { * …)` (another tab, a programmatic write) propagate into the open editor. * * - **apply-effect**: reflects external `handle.draft` changes into the form. + * - **absorb-effect**: folds the form's own settling into the baseline until + * the user's first input, so a schema that moved on is not a draft. * - **persist-effect**: writes form edits back through the handle, dropping - * the draft when the form is back at the deployed baseline. + * the draft when the form is back at the baseline. * * Both effect bodies are `untrack`ed and gated by `cfgDiffers` * idempotence so they can't feed back into each other. Must be called once @@ -99,14 +113,56 @@ export function useTriggerDraftSync(opts: TriggerDraftSyncOptions): TriggerDraft }) const handle = $derived(handles[0]) + // Gated until the user puts something in (see `onUserInput`): the baseline + // absorbs whatever the form settles on and nothing persists, so an untouched + // trigger never reports unsaved changes. A drawer opened ON a restored draft + // absorbs nothing — that divergence is the user's own. + let settledBaseline: Cfg | undefined = $state(undefined) + let userEdited = $state(false) + let openedOnDraft = $state(false) + // These editors are mounted by the list page, not by the drawer, so input + // arriving while the drawer is still loading is the click that opened it + // (or anything else on the page behind) — never an edit to this form. + onUserInput(() => { + if (!opts.drawerLoading()) userEdited = true + }) + + /** The deployed config, plus whatever the form settled on by itself. */ + const baseline = $derived(settledBaseline ?? opts.deployed()) + + $effect(() => { + // A reload re-opens the gate's window: the drawer is being pointed at a + // different trigger, or the same one re-read from the backend. The click + // that opened it landed before this, hence the reset of `userEdited`. + if (!opts.drawerLoading()) return + untrack(() => { + settledBaseline = undefined + userEdited = false + openedOnDraft = false + }) + }) + + // absorb-effect: pre-edit form drift joins the baseline. + $effect(() => { + if (opts.drawerLoading() || userEdited || openedOnDraft) return + const cfg = opts.getCfg() + const deployed = opts.deployed() + if (cfg == null || deployed == null) return + // Snapshot before untracking: `getCfg` hands back the form's `$state` + // objects by reference, so only a deep read subscribes to the nested + // writes the form makes as it settles. + const settled = snapshotCfg(cfg) + untrack(() => { + if (cfgDiffers(settled, settledBaseline ?? deployed)) settledBaseline = settled + }) + }) + // Live "is there a local draft?" — the form diverges from the deployed // baseline. Gated on `!drawerLoading` (the baseline isn't settled yet // mid-load) and on a non-null baseline (a brand-new trigger has none, so // "unsaved changes" / discard-to-deployed is meaningless there). const hasDraft = $derived( - !opts.drawerLoading() && - opts.deployed() != null && - cfgDiffers(opts.getCfg() as Cfg, opts.deployed() as Cfg) + !opts.drawerLoading() && baseline != null && cfgDiffers(opts.getCfg() as Cfg, baseline as Cfg) ) // Reactive "banner is possible" — depends on `drawerLoading()` so it @@ -131,7 +187,7 @@ export function useTriggerDraftSync(opts: TriggerDraftSyncOptions): TriggerDraft if (opts.drawerLoading() || d == null) return untrack(() => { if (cfgDiffers(d, opts.getCfg() as Cfg)) { - void opts.applyCfg(d) + void opts.applyCfg(snapshotCfg(d)) } }) }) @@ -148,30 +204,32 @@ export function useTriggerDraftSync(opts: TriggerDraftSyncOptions): TriggerDraft discardTimer = undefined if (opts.drawerLoading()) return const cfg = opts.getCfg() - const deployed = opts.deployed() const h = handle if (!h || cfg == null) return - if (!cfgDiffers(cfg, deployed) && cfgDiffers(h.draft, deployed)) { - discard(opts.path(), deployed, true) + if (!cfgDiffers(cfg, baseline) && cfgDiffers(h.draft, baseline)) { + discard(opts.path(), baseline, true) } }, 600) } // persist-effect: form edits → handle; drop the draft when back at the - // deployed baseline. + // baseline. $effect(() => { if (opts.drawerLoading() || !opts.path()) return + // Nothing persists before the user's first input — the form's own + // settling is not an edit, and gating here rather than relying on the + // absorb-effect having run first keeps the two effects order-independent. + if (!userEdited && !openedOnDraft) return const cfg = opts.getCfg() if (cfg == null) return untrack(() => { const h = handle if (!h) return - const deployed = opts.deployed() - if (cfgDiffers(cfg, deployed)) { + if (cfgDiffers(cfg, baseline)) { if (cfgDiffers(cfg, h.draft)) h.draft = cfg - } else if (cfgDiffers(h.draft, deployed)) { - // Only when a draft actually exists to drop: `h.draft` equals - // `deployed` right after a discard or the post-load seed. + } else if (cfgDiffers(h.draft, baseline)) { + // Only when a draft actually exists to drop: `h.draft` equals the + // baseline right after a discard or the post-load seed. scheduleAutoDiscard() } }) @@ -202,7 +260,7 @@ export function useTriggerDraftSync(opts: TriggerDraftSyncOptions): TriggerDraft return hasBaseline }, get deployed() { - return opts.deployed() + return baseline }, get current() { return opts.getCfg() @@ -211,8 +269,14 @@ export function useTriggerDraftSync(opts: TriggerDraftSyncOptions): TriggerDraft const d = handle?.draft if (cfgDiffers(d, opts.getCfg() as Cfg)) { // Overlay the local autosave on the just-loaded backend config. - await opts.applyCfg(d) + await opts.applyCfg(snapshotCfg(d)) } + // The form is not rendered while the drawer loads, so anything that + // diverges from the deployed config right now is a draft restored onto + // it — by the overlay above, or by the editor from the backend before + // calling this — never the form settling. Absorbing that into the + // baseline would hide the user's own work behind a clean drawer. + openedOnDraft = cfgDiffers(opts.getCfg() as Cfg, opts.deployed()) // Adopt the post-load form state as the cell's baseline without // POSTing, consuming the entry's one-shot first-write seed guard. // Trigger drawers never write the cell programmatically on open, so @@ -221,14 +285,20 @@ export function useTriggerDraftSync(opts: TriggerDraftSyncOptions): TriggerDraft const p = opts.path() const cfg = opts.getCfg() if (ws && p && cfg != null) { - UserDraft.seed(opts.itemKind, p, structuredClone($state.snapshot(cfg)) as Cfg, { + UserDraft.seed(opts.itemKind, p, snapshotCfg(cfg) as Cfg, { workspace: ws }) } }, async resetToDeployed(path: string) { - const deployedCfg = structuredClone($state.snapshot(opts.deployed())) as Cfg + const deployedCfg = snapshotCfg(opts.deployed()) as Cfg discard(path, deployedCfg) + // Nothing of the user's is left in the form, so the gate closes again + // — otherwise the form settles on the schema's values a second time + // and the discarded draft comes straight back. + settledBaseline = undefined + userEdited = false + openedOnDraft = false await opts.applyCfg(deployedCfg) }, discard diff --git a/frontend/src/lib/userDraft.svelte.ts b/frontend/src/lib/userDraft.svelte.ts index 7942694f31..57e911dd3c 100644 --- a/frontend/src/lib/userDraft.svelte.ts +++ b/frontend/src/lib/userDraft.svelte.ts @@ -219,7 +219,10 @@ const DRAFT_COMPARE_IGNORED_FIELDS = [ 'version_id', 'parent_version', 'is_draft', - 'assets' + 'assets', + // Fixed at creation and absent from the resource editor's draft shape, so + // it only ever shows up on the deployed side of a comparison. + 'resource_type' ] as const /** diff --git a/frontend/src/lib/userDraftEditGate.ts b/frontend/src/lib/userDraftEditGate.ts new file mode 100644 index 0000000000..e3057a73ee --- /dev/null +++ b/frontend/src/lib/userDraftEditGate.ts @@ -0,0 +1,66 @@ +import { onDestroy } from 'svelte' + +/** + * What kind of event opened the gate. + * + * - `value`: the user changed something — the event *is* the edit. + * - `precursor`: a gesture that usually precedes an edit. Needed because a + * custom component (a picker, a toggle built out of divs) writes its value + * through Svelte state and fires no native value event at all, so waiting for + * one would drop those edits. The cost is that a bare click counts too. + */ +export type UserInputKind = 'value' | 'precursor' + +/** Events that ARE an edit. `drop` and `paste` matter on their own: text + * dragged in from another application produces no pointer or key event in this + * document at all. */ +const VALUE_EVENTS = ['input', 'change', 'drop', 'paste'] as const +/** `click` is here for the controls that mutate state from a click handler and + * fire no native value event — ArgInput's "Add item", say. A mouse always sends + * `pointerdown` first, but an assistive technology can activate one with a + * trusted `click` alone, and that is a real edit with nothing else to catch it. */ +const PRECURSOR_EVENTS = ['pointerdown', 'keydown', 'click'] as const + +/** + * A draft is supposed to record what the USER changed, but an editor built + * from a schema writes into the value on its own: the form materializes a + * property the stored item never carried (an empty string, `false`, the first + * option of a required enum, a schema `default`) and deletes one a `showExpr` + * hides. So merely opening an item whose schema has moved on makes it diverge + * from the deployed value with nobody having touched it — a draft nobody asked + * for, cluttering the workspace. + * + * An editor guards against that by gating its draft on this: nothing the form + * settles on counts until the user has actually put something in. Callers + * decide what a gate covers (the resource editor keys it by workspace, since + * switching workspaces re-renders the form against a fresh value) and what + * gating means for them — suspending the autosave, absorbing the settled value + * into the deployed baseline, or both. + * + * Capture phase puts this ahead of the handler that writes the value, so a gate + * opened here is already open by the time the edit lands. Listening on the + * document rather than the editor's own subtree is deliberate: pickers and + * modals render in portals outside it, and missing a real edit would silently + * drop the user's work, while opening the gate too eagerly only costs the + * phantom draft that existed before. + * + * Registers for the lifetime of the calling component — call it during init. + */ +export function onUserInput(handle: (kind: UserInputKind) => void): void { + if (typeof document === 'undefined') return + const listeners: Array<[string, (e: Event) => void]> = [] + const register = (type: string, kind: UserInputKind) => { + const onEvent = (e: Event) => { + // A programmatic `dispatchEvent` is untrusted, which is what keeps the + // form's own settling from opening the gate it is gated by. + if (e.isTrusted) handle(kind) + } + document.addEventListener(type, onEvent, true) + listeners.push([type, onEvent]) + } + for (const type of VALUE_EVENTS) register(type, 'value') + for (const type of PRECURSOR_EVENTS) register(type, 'precursor') + onDestroy(() => { + for (const [type, onEvent] of listeners) document.removeEventListener(type, onEvent, true) + }) +} diff --git a/frontend/src/lib/userDraftPrune.test.ts b/frontend/src/lib/userDraftPrune.test.ts new file mode 100644 index 0000000000..0276598d45 --- /dev/null +++ b/frontend/src/lib/userDraftPrune.test.ts @@ -0,0 +1,243 @@ +import { describe, it, expect, beforeEach, vi } from 'vitest' + +// The sweep DELETES drafts, so the guard that matters is which rows it picks. +// Stub the two collaborators it decides from — the draft listing and the +// per-kind diff — and assert on what it discards. +const listDrafts = vi.fn() +const getDraftDiffValues = vi.fn() +const updateDraft = vi.fn(async () => ({ status: 'saved', current_timestamp: 'x' })) + +vi.mock('./gen', () => ({ + DraftService: { + listDrafts: (...a: unknown[]) => listDrafts(...(a as [])), + updateDraft: (...a: unknown[]) => updateDraft(...(a as [])) + } +})) +// Only `getDraftDiffValues` is stubbed; `canDiffDraftKind` is the real one, so +// the kind filter is pinned against the actual overlay table. +vi.mock('./utils_draft_deploy', async (orig) => ({ + ...(await orig>()), + getDraftDiffValues: (...a: unknown[]) => getDraftDiffValues(...(a as [])) +})) +vi.mock('./localDraftHints.svelte', () => ({ setLocalDraftHint: vi.fn() })) +vi.mock('./workspaceDrafts.svelte', () => ({ invalidateWorkspaceDrafts: vi.fn() })) +const sendUserToast = vi.fn() +vi.mock('./toast', () => ({ sendUserToast: (...a: unknown[]) => sendUserToast(...(a as [])) })) + +// The sweep reads exactly one thing from the syncer — whether this tab is +// mid-write on the key — and writes nothing back to it. +let syncState = 'none' +vi.mock('./userDraftDbSyncer.svelte', () => ({ + UserDraftDbSyncer: { + getState: () => ({ + get state() { + return syncState + } + }) + } +})) + +let liveDraft = false +vi.mock('./userDraft.svelte', async (orig) => ({ + ...(await orig>()), + UserDraft: { has: () => liveDraft } +})) + +import { pruneMeaninglessDrafts } from './userDraftPrune' + +const row = (over: Record = {}) => ({ + kind: 'resource', + path: 'u/me/r', + draft_only: false, + legacy_draft: false, + mine: true, + can_write: true, + created_at: '2026-01-01T00:00:00Z', + ...over +}) +const diff = (over: Record = {}) => ({ + deployed: { value: { host: 'h' } }, + draft: { value: { host: 'h' } }, + hasDraft: true, + noDeployed: false, + ...over +}) +const discardedPaths = () => updateDraft.mock.calls.map((c: any[]) => c[0].path as string) + +beforeEach(() => { + localStorage.clear() + vi.clearAllMocks() + updateDraft.mockResolvedValue({ status: 'saved', current_timestamp: 'x' }) + syncState = 'none' + liveDraft = false +}) + +describe('pruneMeaninglessDrafts', () => { + it('discards a draft whose diff against the deployed value is empty', async () => { + listDrafts.mockResolvedValue([row()]) + getDraftDiffValues.mockResolvedValue(diff()) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(discardedPaths()).toEqual(['u/me/r']) + }) + + it('keeps a draft that carries a real change', async () => { + listDrafts.mockResolvedValue([row()]) + getDraftDiffValues.mockResolvedValue(diff({ draft: { value: { host: 'other' } } })) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(updateDraft).not.toHaveBeenCalled() + }) + + it('never touches a draft-only item — the draft is the whole item', async () => { + listDrafts.mockResolvedValue([row({ draft_only: true })]) + getDraftDiffValues.mockResolvedValue(diff()) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(updateDraft).not.toHaveBeenCalled() + }) + + it('never touches another user’s row, or one it cannot write', async () => { + listDrafts.mockResolvedValue([row({ mine: false }), row({ path: 'u/me/b', can_write: false })]) + getDraftDiffValues.mockResolvedValue(diff()) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(updateDraft).not.toHaveBeenCalled() + }) + + it('leaves a draft alone when its diff cannot be fetched, and retries it later', async () => { + listDrafts.mockResolvedValue([row()]) + getDraftDiffValues.mockRejectedValue(new Error('boom')) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(updateDraft).not.toHaveBeenCalled() + // A row that could not be judged is not a row that carries changes, so + // the pass must stay open rather than strand it. + getDraftDiffValues.mockResolvedValue(diff()) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(discardedPaths()).toEqual(['u/me/r']) + }) + + it('conditions the delete on the timestamp it judged, so a row that moved is spared', async () => { + listDrafts.mockResolvedValue([row()]) + getDraftDiffValues.mockResolvedValue(diff()) + updateDraft.mockResolvedValue({ status: 'conflict', current_timestamp: 'newer' }) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(updateDraft).toHaveBeenCalledWith({ + workspace: 'main', + kind: 'resource', + path: 'u/me/r', + requestBody: { value: null, last_sync: '2026-01-01T00:00:00Z', force: false } + }) + // Refused, so nothing is reported as cleared — and the sweep leaves no + // state behind for the editor's own autosave to trip over. + expect(sendUserToast).not.toHaveBeenCalled() + }) + + it('does not keep retrying a row the server will never judge', async () => { + listDrafts.mockResolvedValue([row()]) + getDraftDiffValues.mockRejectedValue({ status: 404 }) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(updateDraft).not.toHaveBeenCalled() + // Sealed: a 4xx is final, unlike the transient case above. + listDrafts.mockResolvedValue([row()]) + getDraftDiffValues.mockResolvedValue(diff()) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(updateDraft).not.toHaveBeenCalled() + }) + + it('gives up after a bounded number of unresolved passes', async () => { + listDrafts.mockResolvedValue([row()]) + getDraftDiffValues.mockRejectedValue(new Error('network')) + for (let i = 0; i < 3; i++) await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(listDrafts).toHaveBeenCalledTimes(3) + // Sealed on the third: an unresolvable row cannot re-list forever. + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(listDrafts).toHaveBeenCalledTimes(3) + }) + + it('skips a kind no diff can be computed for, and still seals', async () => { + listDrafts.mockResolvedValue([row({ kind: 'trigger_webhook', path: 'u/me/hook' })]) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(getDraftDiffValues).not.toHaveBeenCalled() + // Unjudgeable is permanent, not transient: leaving the pass open would + // re-run the sweep on every page load forever. + listDrafts.mockResolvedValue([row()]) + getDraftDiffValues.mockResolvedValue(diff()) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(updateDraft).not.toHaveBeenCalled() + }) + + it('leaves alone a draft this tab is editing', async () => { + liveDraft = true + listDrafts.mockResolvedValue([row()]) + getDraftDiffValues.mockResolvedValue(diff()) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(updateDraft).not.toHaveBeenCalled() + }) + + it('leaves alone a draft with a write queued or in flight', async () => { + syncState = 'pending' + listDrafts.mockResolvedValue([row()]) + getDraftDiffValues.mockResolvedValue(diff()) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(updateDraft).not.toHaveBeenCalled() + }) + + it('does not count, or seal the pass on, a delete that failed to send', async () => { + listDrafts.mockResolvedValue([row()]) + getDraftDiffValues.mockResolvedValue(diff()) + updateDraft.mockRejectedValueOnce(new Error('network')) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(sendUserToast).not.toHaveBeenCalled() + // The pass stayed open, so the draft left behind is retried. + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(discardedPaths()).toEqual(['u/me/r', 'u/me/r']) + }) + + it('never touches a legacy workspace-level row', async () => { + listDrafts.mockResolvedValue([row({ legacy_draft: true })]) + getDraftDiffValues.mockResolvedValue(diff()) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(updateDraft).not.toHaveBeenCalled() + }) + + it('only sweeps the kinds whose editors are gated', async () => { + listDrafts.mockResolvedValue([ + row({ kind: 'script', path: 'u/me/s' }), + row({ kind: 'flow', path: 'u/me/f' }), + row({ kind: 'app', path: 'u/me/a' }), + row({ kind: 'trigger_schedule', path: 'u/me/sched' }), + row() + ]) + getDraftDiffValues.mockResolvedValue(diff()) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(discardedPaths().sort()).toEqual(['u/me/r', 'u/me/sched']) + // The expensive payload fetches are never made for the ungated kinds. + expect(getDraftDiffValues).toHaveBeenCalledTimes(2) + }) + + it('leaves the pass open when a row was skipped as busy', async () => { + liveDraft = true + listDrafts.mockResolvedValue([row()]) + getDraftDiffValues.mockResolvedValue(diff()) + await pruneMeaninglessDrafts('main', 'me@x.dev') + liveDraft = false + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(discardedPaths()).toEqual(['u/me/r']) + }) + + it('runs once per workspace and user', async () => { + listDrafts.mockResolvedValue([row()]) + getDraftDiffValues.mockResolvedValue(diff()) + await pruneMeaninglessDrafts('main', 'me@x.dev') + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(updateDraft).toHaveBeenCalledTimes(1) + await pruneMeaninglessDrafts('other', 'me@x.dev') + expect(updateDraft).toHaveBeenCalledTimes(2) + }) + + it('retries next mount when the listing failed', async () => { + listDrafts.mockRejectedValueOnce(new Error('offline')) + await pruneMeaninglessDrafts('main', 'me@x.dev') + listDrafts.mockResolvedValue([row()]) + getDraftDiffValues.mockResolvedValue(diff()) + await pruneMeaninglessDrafts('main', 'me@x.dev') + expect(discardedPaths()).toEqual(['u/me/r']) + }) +}) diff --git a/frontend/src/lib/userDraftPrune.ts b/frontend/src/lib/userDraftPrune.ts new file mode 100644 index 0000000000..465048d09d --- /dev/null +++ b/frontend/src/lib/userDraftPrune.ts @@ -0,0 +1,230 @@ +/** + * One-off sweep that drops drafts carrying no changes. + * + * `onUserInput` stops new ones from being written; the ones already stored need + * this pass to clear. Runs once per (workspace, user) per browser, after the + * localStorage→DB migration so anything it just uploaded is swept too. + * + * Scoped to the kinds whose editors that gate covers. A script, flow or app + * draft only ever came from an explicit edit, so there is no phantom to clear + * there — and `getDraftDiffValues` would fetch each one's full deployed payload + * at login to prove it. + * + * A draft is dropped only when the diff the user would be shown is empty: both + * sides come from `getDraftDiffValues`, the same canonicalization the diff + * drawer renders, compared with the same `draftValuesEqual` the editors use. + * Anything that can't be established is left alone — a `draft_only` item (no + * deployed counterpart, so discarding would destroy the item itself), a kind + * with no diff support, a failed fetch, and the legacy workspace-level rows, + * which belong to nobody and are admin-gated to migrate. + * + * Deleting is the dangerous half, and the equality behind it is always stale: + * it was read one round trip ago, and every candidate is read before any is + * deleted. So the delete is a compare-and-delete — `last_sync` is the + * timestamp the row was judged on, and the backend drops it only if nothing has + * written it since, whoever wrote it. + * + * It is sent straight to `DraftService`, NOT through `UserDraftDbSyncer`. That + * syncer exists to autosave an editor's own live value, and everything it does + * for that — parking the payload for the `pagehide` flush, debouncing, holding + * a per-tab `last_sync` baseline and conflict state — is a way for a one-shot + * delete to reach back into whatever the user is doing in the same tab. This + * sweep wants exactly one conditional request and no state afterwards. + */ + +import { DraftService } from './gen' +import type { UserDraftItemKind } from './gen' +import { sendUserToast } from './toast' +import { setLocalDraftHint } from './localDraftHints.svelte' +import { UserDraft, draftValuesEqual } from './userDraft.svelte' +import { UserDraftDbSyncer } from './userDraftDbSyncer.svelte' +import { canDiffDraftKind, getDraftDiffValues } from './utils_draft_deploy' +import { invalidateWorkspaceDrafts } from './workspaceDrafts.svelte' + +const SENTINEL_PREFIX = 'userdraft/pruned/v1/' + +/** A pass that leaves anything unresolved runs again next mount, which costs a + * listing plus an overlay GET per row. Bounded so no permanently-unresolvable + * row can make that repeat forever — whatever the reason it can't be judged. */ +const MAX_PASSES = 3 + +/** The editors whose forms are built from a schema, and so the only kinds that + * could have banked a draft nobody wrote — minus the ones no diff can be + * computed for, which would throw and keep the pass unsealed forever. */ +function isSweepableKind(kind: UserDraftItemKind): boolean { + return (kind === 'resource' || kind.startsWith('trigger_')) && canDiffDraftKind(kind) +} + +/** Overlay GETs are one round trip each and a cluttered workspace has dozens; + * a small window keeps the sweep off the critical path of a fresh login. */ +const CONCURRENCY = 4 + +/** Guards against the layout effect firing again before the sentinel lands. */ +const inFlight = new Set() + +type Candidate = { + kind: UserDraftItemKind + path: string + /** The row's `created_at` as listed — the baseline the delete is conditioned on. */ + createdAt: string +} + +const attemptsKey = (sentinel: string) => `${sentinel}:attempts` + +function readAttempts(sentinel: string): number { + try { + const n = Number(localStorage.getItem(attemptsKey(sentinel))) + return Number.isFinite(n) && n > 0 ? n : 0 + } catch { + return 0 + } +} + +/** Is this tab holding or writing this draft right now? */ +function busyLocally(workspace: string, kind: UserDraftItemKind, path: string): boolean { + if (UserDraft.has(kind, path, { workspace })) return true + return UserDraftDbSyncer.getState({ workspace, itemKind: kind, path }).state !== 'none' +} + +/** A 4xx is the server's final answer for this row — the item is gone, or the + * kind's overlay endpoint isn't served by this build (a feature-gated trigger + * on CE). Retrying it on every page load would never succeed. Anything else + * (network, 5xx) is worth another pass. 429 asks for exactly that. */ +function isPermanentlyUnjudgeable(e: unknown): boolean { + const status = (e as { status?: unknown })?.status + return typeof status === 'number' && status >= 400 && status < 500 && status !== 429 +} + +/** `undefined` when the diff could not be fetched and might be next time — + * distinct from `false`, so the caller can leave the pass open rather than + * strand a row it never judged. */ +async function carriesNoChanges( + workspace: string, + { kind, path }: Candidate +): Promise { + try { + const { deployed, draft, hasDraft, noDeployed } = await getDraftDiffValues( + kind, + path, + workspace + ) + // `hasDraft` false means the overlay had no draft row and the item's own + // value stood in for the draft side — there is nothing to discard, and the + // two sides would compare equal by construction. + if (!hasDraft || noDeployed) return false + return draftValuesEqual(draft, deployed) + } catch (e) { + return isPermanentlyUnjudgeable(e) ? false : undefined + } +} + +async function mapWithLimit( + items: T[], + limit: number, + fn: (item: T) => Promise +): Promise { + const out = new Array(items.length) + let next = 0 + await Promise.all( + Array.from({ length: Math.min(limit, items.length) }, async () => { + while (next < items.length) { + const i = next++ + out[i] = await fn(items[i]) + } + }) + ) + return out +} + +export async function pruneMeaninglessDrafts(workspace: string, userKey: string): Promise { + if (typeof localStorage === 'undefined') return + const sentinel = `${SENTINEL_PREFIX}${workspace}/${userKey}` + if (inFlight.has(sentinel)) return + try { + if (localStorage.getItem(sentinel)) return + } catch { + // Storage unavailable (private mode): the sweep can't record that it ran, + // and re-running it on every mount would cost an overlay GET per draft. + return + } + inFlight.add(sentinel) + try { + const rows = await DraftService.listDrafts({ workspace }) + // Anything left unresolved keeps the pass open: a row skipped as busy was + // never judged, and one whose delete failed is still there. Sealing on + // either would strand it. + let unresolved = 0 + const candidates: Candidate[] = rows + // `draft_only` rows ARE the item; `mine` / `can_write` are the same + // gate the discard endpoint enforces, so anything else would 403. + .filter((r) => !r.draft_only && r.mine && r.can_write) + .filter((r) => isSweepableKind(r.kind) && !r.legacy_draft) + .filter((r) => { + if (!busyLocally(workspace, r.kind, r.path)) return true + unresolved++ + return false + }) + .map((r) => ({ + kind: r.kind, + path: r.path, + createdAt: r.created_at + })) + + const empty: Candidate[] = [] + await mapWithLimit(candidates, CONCURRENCY, async (c) => { + const verdict = await carriesNoChanges(workspace, c) + if (verdict === undefined) unresolved++ + else if (verdict) empty.push(c) + }) + + let discarded = 0 + for (const c of empty) { + // Re-check: the reads above took a while, and the user may have opened + // this item in the meantime. + if (busyLocally(workspace, c.kind, c.path)) { + unresolved++ + continue + } + try { + const resp = await DraftService.updateDraft({ + workspace, + kind: c.kind, + path: c.path, + requestBody: { value: null, last_sync: c.createdAt, force: false } + }) + // `conflict` means the row moved past the timestamp we judged it on, + // so it is no longer the empty draft we decided to drop. + if (resp.status === 'saved') { + setLocalDraftHint(workspace, c.kind, c.path, false) + discarded++ + } + } catch { + unresolved++ + } + } + if (discarded > 0) { + invalidateWorkspaceDrafts(workspace) + sendUserToast(`Cleared ${discarded} draft${discarded > 1 ? 's' : ''} that carried no changes`) + } + // Seal once nothing is left hanging, or once we have tried enough times + // that whatever is hanging is not going to resolve. + const attempts = readAttempts(sentinel) + 1 + if (unresolved === 0 || attempts >= MAX_PASSES) { + try { + localStorage.setItem(sentinel, new Date().toISOString()) + localStorage.removeItem(attemptsKey(sentinel)) + } catch { + // Nothing to do — the pass is idempotent, it just runs again. + } + } else { + try { + localStorage.setItem(attemptsKey(sentinel), String(attempts)) + } catch {} + } + } catch { + // Fire-and-forget from the layout: a workspace whose draft list can't be + // read is left exactly as it was. + } finally { + inFlight.delete(sentinel) + } +} diff --git a/frontend/src/lib/utils_draft_deploy.ts b/frontend/src/lib/utils_draft_deploy.ts index f322bb27b9..0d121d0627 100644 --- a/frontend/src/lib/utils_draft_deploy.ts +++ b/frontend/src/lib/utils_draft_deploy.ts @@ -95,6 +95,19 @@ const OVERLAY_GETTERS: Partial< EmailTriggerService.getEmailTrigger({ workspace, path, getDraft: true }) } +/** Whether `getDraftDiffValues` can produce a diff for this kind at all. The + * script/flow/app family is handled inline; every other kind needs an overlay + * getter and throws without one — several trigger kinds have none. */ +export function canDiffDraftKind(kind: DraftKind): boolean { + return ( + kind === 'script' || + kind === 'flow' || + kind === 'app' || + kind === 'raw_app' || + OVERLAY_GETTERS[kind] !== undefined + ) +} + /** Strip the per-user draft-overlay metadata, returning `{deployed, draft}`. */ function splitOverlay(r: any): { deployed: any diff --git a/frontend/src/routes/(root)/(logged)/+layout.svelte b/frontend/src/routes/(root)/(logged)/+layout.svelte index a20c61f2ff..e82d328c76 100644 --- a/frontend/src/routes/(root)/(logged)/+layout.svelte +++ b/frontend/src/routes/(root)/(logged)/+layout.svelte @@ -74,6 +74,7 @@ import { createUsageResources, registerUsageResources } from '$lib/usage.svelte' import { purgeLegacyUserDrafts } from '$lib/userDraftLegacyMigration' import { migrateUserDraftsToDb } from '$lib/userDraftDbMigration' + import { pruneMeaninglessDrafts } from '$lib/userDraftPrune' import DraftMigrationErrorModal from '$lib/components/DraftMigrationErrorModal.svelte' import { onDestroy, setContext, untrack } from 'svelte' import { base } from '$app/paths' @@ -792,12 +793,16 @@ // drafts). `migrateUserDraftsToDb` then pushes the workspace-scoped // `userdraft/w/{ws}/{kind}/{path}` keys — written by the editor with the // correct workspace — onto the server-side draft table, clearing LS on - // success. + // success. `pruneMeaninglessDrafts` then clears the drafts an older, stricter + // comparison saved for changes nobody made; it runs after the upload so the + // entries that just landed are swept in the same pass. $effect(() => { - if ($workspaceStore && $userStore) { + const ws = $workspaceStore + const email = $userStore?.email + if (ws && email) { untrack(() => { purgeLegacyUserDrafts() - void migrateUserDraftsToDb() + void migrateUserDraftsToDb().then(() => pruneMeaninglessDrafts(ws, email)) }) } }) From a9d42b489f06261919ba01e8221272e3ffcf6b22 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 7 Sep 2026 18:20:28 +0000 Subject: [PATCH 15/15] chore(main): release 1.805.0 (#10995) * chore(main): release 1.805.0 * Apply automatic changes --------- Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com> --- .release-please-manifest.json | 2 +- CHANGELOG.md | 25 +++ backend/Cargo.lock | 202 +++++++++--------- backend/Cargo.toml | 4 +- .../parsers/windmill-parser-wasm/Cargo.lock | 48 ++--- .../parsers/windmill-parser-wasm/Cargo.toml | 2 +- backend/windmill-api/openapi.yaml | 2 +- benchmarks/lib.ts | 2 +- cli/src/core/constants.ts | 2 +- frontend/package-lock.json | 4 +- frontend/package.json | 2 +- lsp/Pipfile | 2 +- openflow.openapi.yaml | 2 +- .../WindmillClient/WindmillClient.psd1 | 2 +- python-client/wmill/pyproject.toml | 2 +- typescript-client/jsr.json | 2 +- typescript-client/package.json | 2 +- version.txt | 2 +- windmill-yaml-validator/package-lock.json | 4 +- windmill-yaml-validator/package.json | 2 +- 20 files changed, 170 insertions(+), 145 deletions(-) diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 5dc2004c6f..f4fac9d2e0 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "1.804.0" + ".": "1.805.0" } diff --git a/CHANGELOG.md b/CHANGELOG.md index 4acfb0edad..f431b9fd96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,30 @@ # Changelog +## [1.805.0](https://github.com/windmill-labs/windmill/compare/v1.804.0...v1.805.0) (2026-09-07) + + +### Features + +* **git-sync:** sync extra_perms for variables ([#11004](https://github.com/windmill-labs/windmill/issues/11004)) ([ee9e550](https://github.com/windmill-labs/windmill/commit/ee9e550a484fda286eeab43b7db5f314b8b2d0d9)) +* go to referenced row from foreign-keyed cells in the database manager ([#10998](https://github.com/windmill-labs/windmill/issues/10998)) ([e2b63d1](https://github.com/windmill-labs/windmill/commit/e2b63d177ae4e5c980cb5da34154540c90771b63)) +* let `// materialize` declare a `dbt://` warehouse-relation write ([#10978](https://github.com/windmill-labs/windmill/issues/10978)) ([c6e0302](https://github.com/windmill-labs/windmill/commit/c6e0302d7c1c60147f19d55a3923be8b1aa99c9c)) +* report resource type picks to the hub and rank pickers by popularity ([#10982](https://github.com/windmill-labs/windmill/issues/10982)) ([48a5615](https://github.com/windmill-labs/windmill/commit/48a56158c135c3b13a02f73b7b8438bc691f85b4)) +* run a linked AI agent's draft when testing a flow, and offer to deploy it ([#10993](https://github.com/windmill-labs/windmill/issues/10993)) ([7feaf61](https://github.com/windmill-labs/windmill/commit/7feaf619cf0ec021d66be14ef535cf2149bec58a)) +* show the new-tab icon on a chat path pill while the modifier is held ([#10976](https://github.com/windmill-labs/windmill/issues/10976)) ([5da4ea4](https://github.com/windmill-labs/windmill/commit/5da4ea43fbd01e43aa14e75dc597d7ce5d8797ab)) + + +### Bug Fixes + +* **cli:** keep permissioned_as on single-item push, as sync push does ([#11000](https://github.com/windmill-labs/windmill/issues/11000)) ([5f3f99b](https://github.com/windmill-labs/windmill/commit/5f3f99ba6915b7c5df663a30b35f4cd02050e728)) +* **cli:** say which workspace id is targeted, and when wmill.yaml is bypassed ([#11006](https://github.com/windmill-labs/windmill/issues/11006)) ([7643e9b](https://github.com/windmill-labs/windmill/commit/7643e9bd77c56f72596b8dca50801baf58984198)) +* **frontend:** no phantom draft when opening a CLI-pushed script ([#10997](https://github.com/windmill-labs/windmill/issues/10997)) ([1be390a](https://github.com/windmill-labs/windmill/commit/1be390aa878e15a58f530f3a878e8f9caeb89c43)) +* **frontend:** stop hover flicker on asset nodes shared with an overflow popover ([#10996](https://github.com/windmill-labs/windmill/issues/10996)) ([519a5c8](https://github.com/windmill-labs/windmill/commit/519a5c8bc70b44a7417e83c26c7b9c58b2c4fb9c)) +* let a draft-only schedule, trigger or resource be deleted ([#11010](https://github.com/windmill-labs/windmill/issues/11010)) ([8d0f475](https://github.com/windmill-labs/windmill/commit/8d0f4754e4e0c78696ee0c97ff2de3016ece3bac)) +* point the app viewer's edit button at the editor for the app's kind ([#11009](https://github.com/windmill-labs/windmill/issues/11009)) ([8f553ea](https://github.com/windmill-labs/windmill/commit/8f553eab353103fd8a28a00532e1766f133590de)) +* seed runs page filter defaults through the url so they survive sync ([#11005](https://github.com/windmill-labs/windmill/issues/11005)) ([f381acd](https://github.com/windmill-labs/windmill/commit/f381acdb37f66f5e272bc37938e69f734987d53f)) +* stop an untouched item's form from saving a draft nobody wrote ([#10964](https://github.com/windmill-labs/windmill/issues/10964)) ([c3f7f8a](https://github.com/windmill-labs/windmill/commit/c3f7f8a45830fb548aa628ebf6e2b6c95c6de67f)) +* write and read python job files as utf-8, not the platform locale ([#10994](https://github.com/windmill-labs/windmill/issues/10994)) ([670404f](https://github.com/windmill-labs/windmill/commit/670404ffe27fedc3858b46b0c6b3312fbe175e13)) + ## [1.804.0](https://github.com/windmill-labs/windmill/compare/v1.803.0...v1.804.0) (2026-09-05) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index e4dc875e49..9a458a8713 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -2763,18 +2763,18 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.16" +version = "0.5.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e" +checksum = "98b0cc327b5bc766e7fda9c9260cc0fa81b43a8e240440422dff70788e3f9ef1" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-deque" -version = "0.8.7" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a" dependencies = [ "crossbeam-epoch", "crossbeam-utils", @@ -2782,27 +2782,27 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.20" +version = "0.9.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-queue" -version = "0.3.13" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "803d13fb3b09d88be9f4dbc29062c66b19bf7170867ceb746d2a8689bf6c7a26" +checksum = "03e8bd762f7479489c70ed6c768ddca99d7296857de437a68dcb2a94365b3fae" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.22" +version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" [[package]] name = "crossterm_winapi" @@ -6640,9 +6640,9 @@ dependencies = [ [[package]] name = "io-uring" -version = "0.7.14" +version = "0.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d64d8ca234d152948ceaede1f419b6a83983a5ecccaac05fb337a809c96d3aa6" +checksum = "ed3bd0ecfbb87805f538bb7b32e5239ca0763890c623e349860ecba69469f2bb" dependencies = [ "bitflags 2.13.1", "cfg-if", @@ -6664,9 +6664,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.12.1" +version = "2.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78" +checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" [[package]] name = "ipnetwork" @@ -9077,9 +9077,9 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pest" -version = "2.9.0" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a07a60cc7a4d00c91f95c685609d1d2f79050e6804b70ebedd7650f0b839bcf" +checksum = "6d45aeb61b4bf818e12d4205f2466f8c4748f85f4fce0146d1c03d69d753f0ad" dependencies = [ "memchr", "ucd-trie", @@ -9087,9 +9087,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.9.0" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3a83744a5c8455b8b3e0dc5031362780a347c878bdd11584d1a8984228cc88d" +checksum = "89cc5a242e25ed4e7704d0be240f2cfbe20a8c27e7e252d94835be93d92dc39f" dependencies = [ "pest", "pest_generator", @@ -9097,9 +9097,9 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.9.0" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0cd3451aa3de60d4b9a1e736885e4dea6b31617598026f12256ad566d63304a" +checksum = "7abf21475cc3820fe4b2ca2dc2142902f67a02189f3b5b3a229f4febc01a43e5" dependencies = [ "pest", "pest_meta", @@ -9110,9 +9110,9 @@ dependencies = [ [[package]] name = "pest_meta" -version = "2.9.0" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e04d3a0849e241d7dfce834c83b1c5edc8622009e8dd51a12ba1927c32f05496" +checksum = "adba4db388f687393c18c51348d44a41d870ca9df71a2c98172ea3035dc6936e" dependencies = [ "pest", ] @@ -14747,7 +14747,7 @@ dependencies = [ [[package]] name = "windmill" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-nats", @@ -14835,7 +14835,7 @@ dependencies = [ [[package]] name = "windmill-ai" -version = "1.804.0" +version = "1.805.0" dependencies = [ "async-stream", "async-trait", @@ -14868,7 +14868,7 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.804.0" +version = "1.805.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14881,7 +14881,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "argon2", @@ -15021,7 +15021,7 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.804.0" +version = "1.805.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15044,7 +15044,7 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.804.0" +version = "1.805.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15061,7 +15061,7 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15087,7 +15087,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.804.0" +version = "1.805.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -15097,7 +15097,7 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.804.0" +version = "1.805.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15114,7 +15114,7 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.804.0" +version = "1.805.0" dependencies = [ "axum 0.8.9", "base64 0.22.1", @@ -15136,7 +15136,7 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15159,7 +15159,7 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.804.0" +version = "1.805.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15175,7 +15175,7 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.804.0" +version = "1.805.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15197,7 +15197,7 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.804.0" +version = "1.805.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15218,7 +15218,7 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.804.0" +version = "1.805.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15232,7 +15232,7 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-nats", @@ -15267,7 +15267,7 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15292,7 +15292,7 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.804.0" +version = "1.805.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15320,7 +15320,7 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15342,7 +15342,7 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.804.0" +version = "1.805.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15362,7 +15362,7 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.804.0" +version = "1.805.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15400,7 +15400,7 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15428,7 +15428,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.804.0" +version = "1.805.0" dependencies = [ "lazy_static", "serde", @@ -15440,7 +15440,7 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.804.0" +version = "1.805.0" dependencies = [ "argon2", "axum 0.8.9", @@ -15464,7 +15464,7 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.804.0" +version = "1.805.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15478,7 +15478,7 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.804.0" +version = "1.805.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15513,7 +15513,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.804.0" +version = "1.805.0" dependencies = [ "chrono", "lazy_static", @@ -15527,7 +15527,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15546,7 +15546,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.804.0" +version = "1.805.0" dependencies = [ "aes-gcm", "aho-corasick", @@ -15652,7 +15652,7 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.804.0" +version = "1.805.0" dependencies = [ "chrono", "futures", @@ -15672,7 +15672,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.804.0" +version = "1.805.0" dependencies = [ "regex", "serde", @@ -15687,7 +15687,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "astral-tokio-tar", @@ -15714,7 +15714,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "futures", @@ -15731,7 +15731,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.804.0" +version = "1.805.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -15747,7 +15747,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-trait", @@ -15768,7 +15768,7 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-trait", @@ -15799,7 +15799,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "arc-swap", @@ -15824,7 +15824,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-stream", @@ -15858,7 +15858,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "futures", @@ -15876,7 +15876,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.804.0" +version = "1.805.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -15885,7 +15885,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "lazy_static", @@ -15897,7 +15897,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "serde_json", @@ -15909,7 +15909,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "gosyn", @@ -15921,7 +15921,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "lazy_static", @@ -15933,7 +15933,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "serde_json", @@ -15945,7 +15945,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "nu-parser", @@ -15956,7 +15956,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15967,7 +15967,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15979,7 +15979,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "rustpython-ast", @@ -15990,7 +15990,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-recursion", @@ -16012,7 +16012,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "serde_json", @@ -16024,7 +16024,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "lazy_static", @@ -16038,7 +16038,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -16055,7 +16055,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "lazy_static", @@ -16068,7 +16068,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "serde", @@ -16080,7 +16080,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "lazy_static", @@ -16098,7 +16098,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -16114,7 +16114,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "rustpython-ast", @@ -16130,7 +16130,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "lazy_static", @@ -16144,7 +16144,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-recursion", @@ -16183,7 +16183,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "const_format", @@ -16223,7 +16223,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.804.0" +version = "1.805.0" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -16234,7 +16234,7 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-recursion", @@ -16269,7 +16269,7 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-trait", @@ -16293,7 +16293,7 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-trait", @@ -16326,7 +16326,7 @@ dependencies = [ [[package]] name = "windmill-trigger-amqp" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-trait", @@ -16353,7 +16353,7 @@ dependencies = [ [[package]] name = "windmill-trigger-azure" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-trait", @@ -16386,7 +16386,7 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-trait", @@ -16406,7 +16406,7 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-trait", @@ -16440,7 +16440,7 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-trait", @@ -16476,7 +16476,7 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-trait", @@ -16499,7 +16499,7 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-trait", @@ -16523,7 +16523,7 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-nats", @@ -16547,7 +16547,7 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-trait", @@ -16582,7 +16582,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-trait", @@ -16610,7 +16610,7 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-trait", @@ -16635,7 +16635,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "bitflags 2.13.1", @@ -16654,7 +16654,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-once-cell", @@ -16771,7 +16771,7 @@ dependencies = [ [[package]] name = "windmill-worker-volumes" -version = "1.804.0" +version = "1.805.0" dependencies = [ "bytes", "futures", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 7abb3b92ee..f0bccb6fa8 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.804.0" +version = "1.805.0" authors.workspace = true edition.workspace = true @@ -88,7 +88,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"] [workspace.package] -version = "1.804.0" +version = "1.805.0" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/parsers/windmill-parser-wasm/Cargo.lock b/backend/parsers/windmill-parser-wasm/Cargo.lock index 9ed900214c..13597b06c9 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.lock +++ b/backend/parsers/windmill-parser-wasm/Cargo.lock @@ -6191,7 +6191,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill-common" -version = "1.804.0" +version = "1.805.0" dependencies = [ "aho-corasick", "anyhow", @@ -6274,7 +6274,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.804.0" +version = "1.805.0" dependencies = [ "proc-macro2", "quote", @@ -6286,7 +6286,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.804.0" +version = "1.805.0" dependencies = [ "convert_case", "serde", @@ -6295,7 +6295,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "lazy_static", @@ -6307,7 +6307,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "serde_json", @@ -6319,7 +6319,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "gosyn", @@ -6331,7 +6331,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "lazy_static", @@ -6343,7 +6343,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "serde_json", @@ -6355,7 +6355,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "nu-parser", @@ -6366,7 +6366,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6377,7 +6377,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6389,7 +6389,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6400,7 +6400,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "async-recursion", @@ -6422,7 +6422,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "serde_json", @@ -6434,7 +6434,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "lazy_static", @@ -6448,7 +6448,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "convert_case", @@ -6465,7 +6465,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "lazy_static", @@ -6478,7 +6478,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "serde", @@ -6490,7 +6490,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "lazy_static", @@ -6508,7 +6508,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -6524,7 +6524,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6540,7 +6540,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "getrandom 0.2.17", @@ -6572,7 +6572,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "lazy_static", @@ -6586,7 +6586,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.804.0" +version = "1.805.0" dependencies = [ "anyhow", "bitflags", diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index cbb7102820..7de860c637 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -12,7 +12,7 @@ resolver = "2" members = ["."] [workspace.package] -version = "1.804.0" +version = "1.805.0" edition = "2021" authors = ["Ruben Fiszel "] diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index aeb7d27f36..5c0a6d57c6 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.804.0 + version: 1.805.0 title: Windmill API contact: diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 12930e8de4..2899ff694c 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.804.0"; +export const VERSION = "v1.805.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/src/core/constants.ts b/cli/src/core/constants.ts index 35bd6bc2e7..b5494b112c 100644 --- a/cli/src/core/constants.ts +++ b/cli/src/core/constants.ts @@ -10,4 +10,4 @@ export const WM_FORK_PREFIX = "wm-fork"; // (e.g. utils.ts) can read it without importing main.ts and creating a circular // dependency (main → workspace → utils → main) that triggers a TDZ. // Re-exported from main.ts for backwards compatibility. -export const VERSION = "1.804.0"; +export const VERSION = "1.805.0"; diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 7dde8c270e..a200c878e3 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "@windmill-labs/components", - "version": "1.804.0", + "version": "1.805.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@windmill-labs/components", - "version": "1.804.0", + "version": "1.805.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { diff --git a/frontend/package.json b/frontend/package.json index 593ad0e404..d96e512c00 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "@windmill-labs/components", - "version": "1.804.0", + "version": "1.805.0", "scripts": { "dev": "vite dev", "dev:ui-builder": "mv static/ui_builder static/ui_builder.dev-disabled 2>/dev/null || true ; trap 'mv static/ui_builder.dev-disabled static/ui_builder 2>/dev/null || true' EXIT ; vite dev", diff --git a/lsp/Pipfile b/lsp/Pipfile index 0015ac3510..e1f33d70f0 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,7 +4,7 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.804.0" +wmill = ">=1.805.0" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index 3e2a3950f4..0bbc5a7a37 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: '3.0.3' info: - version: 1.804.0 + version: 1.805.0 title: OpenFlow Spec contact: name: Ruben Fiszel diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index aa3bfc30a3..1068d7b56c 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. - ModuleVersion = '1.804.0' + ModuleVersion = '1.805.0' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index 39d53054cb..d00cd3223f 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.804.0" +version = "1.805.0" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index 7d5452bf42..abbf63201c 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.804.0", + "version": "1.805.0", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./sqlUtils.ts", "./client.ts", "./wacError.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index decf7d342f..1d3542703d 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.804.0", + "version": "1.805.0", "author": "Ruben Fiszel", "license": "Apache 2.0", "homepage": "https://github.com/windmill-labs/windmill/tree/main/typescript-client#readme", diff --git a/version.txt b/version.txt index 494531e9d9..c746f61dea 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.804.0 +1.805.0 diff --git a/windmill-yaml-validator/package-lock.json b/windmill-yaml-validator/package-lock.json index fd43bd29a1..cc63f3a090 100644 --- a/windmill-yaml-validator/package-lock.json +++ b/windmill-yaml-validator/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-yaml-validator", - "version": "1.804.0", + "version": "1.805.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-yaml-validator", - "version": "1.804.0", + "version": "1.805.0", "license": "Apache 2.0", "dependencies": { "@stoplight/yaml": "^4.3.0", diff --git a/windmill-yaml-validator/package.json b/windmill-yaml-validator/package.json index da7c582f9e..d88bcdcf0c 100644 --- a/windmill-yaml-validator/package.json +++ b/windmill-yaml-validator/package.json @@ -1,6 +1,6 @@ { "name": "windmill-yaml-validator", - "version": "1.804.0", + "version": "1.805.0", "description": "YAML validator for Windmill flow, schedule, and trigger files", "main": "dist/index.js", "types": "dist/index.d.ts",
Triggers to deploy
+
+
+ + {#if trigger.isPrimary} + + {/if} +
+ +
+ +
+
+ 		+ {#if permission === 'deploy'} +
+ toggleTrigger(trigger, e.detail)} + > + {#snippet children({ item })} + + + {/snippet} + +
+ {:else if permission === 'admin-only'} + Admin only + {:else if permission === 'invalid-config'} + Invalid config + {/if} +
Agents to deploy
+
+ +
+
+ + + {agent.path} + + {#if agent.noDeployed} + Never deployed + {/if} +
+ {#if agent.noDeployed && !isSelectedAgent} + + + Never deployed, so the flow will not run until this agent is deployed. + + {/if} +
+
+ 		+ {#if permission.state === 'deploy'} +
+ toggleAgent(agent, e.detail)} + > + {#snippet children({ item })} + + + {/snippet} + +
+ {:else if permission.state === 'read-only'} + + Read-only + + {:else} + Invalid config + {/if} +
Triggers to deploy
-
-
- - {#if trigger.isPrimary} - - {/if} -
-
- -
-
- 		- {#if permission === 'deploy'} -
- toggleTrigger(trigger, e.detail)} - > - {#snippet children({ item })} - - - {/snippet} - -
- {:else if permission === 'admin-only'} - Admin only - {:else if permission === 'invalid-config'} - Invalid config - {/if} -
- No draft triggers found -