From 56dd940e34b146c3ca25de7959b4fb618308eb86 Mon Sep 17 00:00:00 2001 From: AlexRV12 <71396855+AlexRV12@users.noreply.github.com> Date: Mon, 14 Sep 2026 15:55:40 +0200 Subject: [PATCH 01/44] fix: keep a script draft's password marking through the chat's run form (#11110) * fix: keep a script draft's password marking through the chat's run form Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Dx9oMWHnKywoqooZBeaPCy * docs: describe what the inferArgs test mock actually lets the suite pin Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Dx9oMWHnKywoqooZBeaPCy --------- Co-authored-by: Claude Opus 5 (1M context) --- .../copilot/chat/global/core.test.ts | 42 ++++++++++++++++++- .../components/copilot/chat/global/core.ts | 22 ++++++---- .../copilot/chat/global/userDraftAdapter.ts | 1 + 3 files changed, 54 insertions(+), 11 deletions(-) diff --git a/frontend/src/lib/components/copilot/chat/global/core.test.ts b/frontend/src/lib/components/copilot/chat/global/core.test.ts index 90ff8a8f77..5b083fd9f7 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.test.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.test.ts @@ -321,8 +321,9 @@ vi.mock('./rawAppBundlerBridge', () => ({ vi.mock('$lib/infer', async () => ({ ...(await vi.importActual('$lib/infer')), - // Avoid the wasm parser in unit tests: the script deploy path infers the arg - // schema but tolerates failure, and these tests don't assert on the schema. + // Avoid the wasm parser in unit tests. A no-op passes the seeded schema through + // untouched, which is what lets the password-marking test pin how a schema is + // seeded in and carried out without pinning inference's own merge rules. inferArgs: vi.fn(async () => {}) })) @@ -4358,6 +4359,43 @@ describe('global AI tools', () => { expect(result).toContain('test logs') }) + // No parser emits `password`, so the stored schema is the only thing carrying it: an edit + // that rewrites the draft's schema from scratch, or a draft read that drops it, unmarks + // the field — and the form then takes the secret as a plain literal into the job's args. + it('test_run_script keeps the password marking of the script it previews', async () => { + vi.mocked(ScriptService.existsScriptByPath).mockResolvedValueOnce(true) + vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce({ + path: 'f/scripts/secretful', + language: 'bun', + schema: { + type: 'object', + properties: { token: { type: 'string', password: true } }, + required: ['token'] + } + } as any) + + await callGlobalTool('write_script', { + path: 'f/scripts/secretful', + language: 'bun', + content: 'export async function main(token: string) { return 1 }' + }) + + let form: any + await callGlobalTool( + 'test_run_script', + { path: 'f/scripts/secretful' }, + { + ...toolCallbacks, + requestRunArgs: async (_toolId, opened) => { + form = opened + return undefined + } + } + ) + + expect(form?.schema?.properties).toMatchObject({ token: { password: true } }) + }) + it('test_run_script previews deployed script content when no draft exists', async () => { vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce({ path: 'f/scripts/deployed-test', diff --git a/frontend/src/lib/components/copilot/chat/global/core.ts b/frontend/src/lib/components/copilot/chat/global/core.ts index 5883c5ae0d..9d459bcf71 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.ts @@ -148,6 +148,7 @@ import type { SessionArtifactsStore } from '../artifacts/artifactsState.svelte' import type { Runnable } from '$lib/components/apps/inputType' import { UserDraft } from '$lib/userDraft.svelte' import { emptySchema } from '$lib/utils' +import type { Schema } from '$lib/common' import { inferArgs } from '$lib/infer' import { resourceRequestSchema, @@ -3424,7 +3425,8 @@ export const globalTools: Tool<{}>[] = [ draftCountByType.set(draft.type, count + 1) byKey.set(getWorkspaceItemKey(draft.type, draft.path, draft.triggerKind), { ...draft, - value: undefined + value: undefined, + schema: undefined }) } } @@ -5033,11 +5035,14 @@ const SCRIPT_SPEC: WriteSpec = { language: args.language, kind: 'script' } - // Infer the arg schema from the content at save time, like the editor does, - // so the persisted draft is the single source of truth at deploy. Keep the - // previous schema (or empty) on failure rather than blanking it. + // Into the schema the base carries, as the editor does at save: `inferArgs` re-seeds + // each arg from the properties it is handed, and those are the only copy of + // `password`, enums, formats and titles — no parser emits them. A clone, so a parse + // failure leaves the previous schema rather than half of one. try { - const schema = emptySchema() + const schema = structuredClone( + draft.schema?.properties ? draft.schema : emptySchema() + ) as Schema await inferArgs(draft.language, draft.content, schema) draft.schema = schema } catch (e) { @@ -5224,10 +5229,9 @@ async function loadScriptForEdit( } } -/** The fields a test form offers, for code that may never have been deployed. A draft the - * chat wrote carries the schema it inferred at write time; anything else — a draft written - * elsewhere, a deployed script whose schema predates an edit — is inferred here from the - * content that is about to run, so the form cannot offer a field the code no longer takes. */ +/** The fields a test form offers, for code that may never have been deployed. The stored + * schema wins wherever it declares fields — a draft's or the deployed script's; only one + * declaring nothing is inferred here, from the content about to run. */ async function schemaForTestRun(script: { content: string language: ScriptLang diff --git a/frontend/src/lib/components/copilot/chat/global/userDraftAdapter.ts b/frontend/src/lib/components/copilot/chat/global/userDraftAdapter.ts index f2c3f02431..7593546114 100644 --- a/frontend/src/lib/components/copilot/chat/global/userDraftAdapter.ts +++ b/frontend/src/lib/components/copilot/chat/global/userDraftAdapter.ts @@ -141,6 +141,7 @@ function scriptDraftToWorkspaceItem(path: string, draft: NewScript): WorkspaceIt summary: draft.summary, language: draft.language, value: draft.content, + schema: draft.schema, parentHash: draft.parent_hash, isDraft: true } From 244ec132914a6e689d7d79da9cf2cb39f920aaef Mon Sep 17 00:00:00 2001 From: Guilhem Date: Mon, 14 Sep 2026 15:55:52 +0200 Subject: [PATCH 02/44] fix(ai-chat): hide other users' MCP servers from the chat unless shared (#11112) * fix(ai-chat): hide other users' MCP servers from the chat unless shared An admin's database role lets the resource listing return every user's u/ MCP resource, so the chat's "+" menu and the assistant settings tab offered servers that carry someone else's credentials. Both lists, and the tool loader behind them, now keep a u/ server only when it belongs to the current user or its extra_perms name them or one of their groups. The Resources page is unchanged. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_0131DsyTiBR6Q54qPXbTX5sC * fix(ai-chat): resolve the MCP viewer for the workspace being listed Username and groups are per workspace, and a session chat can operate on a workspace other than the one being browsed, so the filter now takes its identity from that workspace's whoami rather than from userStore. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_0131DsyTiBR6Q54qPXbTX5sC * fix(ai-chat): list the whole MCP catalog before filtering, one predicate The visibility filter runs after the server's LIMIT, so a 100-row page could drop the viewer's own servers behind foreign u/ rows. The three call sites now ask for 1000 and call the tested predicate directly. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_0131DsyTiBR6Q54qPXbTX5sC --------- Co-authored-by: Claude Fable 5.1 --- .../copilot/chat/AssistantMcpSection.svelte | 19 +++++-- .../copilot/chat/global/mcpTools.test.ts | 2 +- .../copilot/chat/global/mcpTools.ts | 18 ++++--- .../src/lib/components/mcp/mcpMenu.svelte.ts | 26 +++++---- .../src/lib/components/mcp/ownServers.test.ts | 53 +++++++++++++++++++ frontend/src/lib/components/mcp/ownServers.ts | 48 +++++++++++++++++ 6 files changed, 144 insertions(+), 22 deletions(-) create mode 100644 frontend/src/lib/components/mcp/ownServers.test.ts create mode 100644 frontend/src/lib/components/mcp/ownServers.ts diff --git a/frontend/src/lib/components/copilot/chat/AssistantMcpSection.svelte b/frontend/src/lib/components/copilot/chat/AssistantMcpSection.svelte index 0ab2812b2e..c8d360e0f9 100644 --- a/frontend/src/lib/components/copilot/chat/AssistantMcpSection.svelte +++ b/frontend/src/lib/components/copilot/chat/AssistantMcpSection.svelte @@ -14,6 +14,11 @@ switch that decides whether this chat carries its tools. import DropdownV2 from '$lib/components/DropdownV2.svelte' import Toggle from '$lib/components/Toggle.svelte' import { isMcpEnabled, setMcpEnabled } from '$lib/components/mcp/enabledServers' + import { + isOwnOrSharedMcpPath, + MCP_LIST_PER_PAGE, + mcpViewer + } from '$lib/components/mcp/ownServers' import { loadProviderIcon } from '$lib/components/mcp/providerIcon' import { cachedProviderKey, @@ -275,11 +280,15 @@ switch that decides whether this chat carries its tools. loading = true loadError = undefined try { - const resources = await ResourceService.listResource({ - workspace: target, - resourceType: 'mcp', - perPage: 100 - }) + const [listed, viewer] = await Promise.all([ + ResourceService.listResource({ + workspace: target, + resourceType: 'mcp', + perPage: MCP_LIST_PER_PAGE + }), + mcpViewer(target) + ]) + const resources = listed.filter((r) => isOwnOrSharedMcpPath(r.path, r.extra_perms, viewer)) if (seq !== loadSeq) return servers = resources.map((r) => ({ path: r.path, diff --git a/frontend/src/lib/components/copilot/chat/global/mcpTools.test.ts b/frontend/src/lib/components/copilot/chat/global/mcpTools.test.ts index e5d3af8b36..9e1b46589c 100644 --- a/frontend/src/lib/components/copilot/chat/global/mcpTools.test.ts +++ b/frontend/src/lib/components/copilot/chat/global/mcpTools.test.ts @@ -4,7 +4,7 @@ const { getMcpToolsMock, callMcpToolMock, listResourceMock, session } = vi.hoist getMcpToolsMock: vi.fn(), callMcpToolMock: vi.fn(), listResourceMock: vi.fn(), - session: { email: 'first@windmill.dev' } + session: { email: 'first@windmill.dev', workspace_id: 'test-ws', username: 'hugo', pgroups: [] } })) vi.mock('../shared', () => ({ diff --git a/frontend/src/lib/components/copilot/chat/global/mcpTools.ts b/frontend/src/lib/components/copilot/chat/global/mcpTools.ts index 6fd59e4a07..c1db1cac1b 100644 --- a/frontend/src/lib/components/copilot/chat/global/mcpTools.ts +++ b/frontend/src/lib/components/copilot/chat/global/mcpTools.ts @@ -2,6 +2,7 @@ import { z } from 'zod' import { ResourceService, type GetMcpToolsResponse } from '$lib/gen' import { createToolDef, type Tool } from '../shared' import { enabledMcpPaths } from '$lib/components/mcp/enabledServers' +import { isOwnOrSharedMcpPath, MCP_LIST_PER_PAGE, mcpViewer } from '$lib/components/mcp/ownServers' /** * Access to the MCP servers the user has connected (resources of type `mcp`) @@ -91,13 +92,18 @@ export async function loadMcpServers(workspace: string): Promise { const enabled = enabledMcpPaths(workspace) if (enabled.length === 0) return [] try { - const resources = await ResourceService.listResource({ - workspace, - resourceType: 'mcp', - perPage: 100 - }) + const [resources, viewer] = await Promise.all([ + ResourceService.listResource({ + workspace, + resourceType: 'mcp', + perPage: MCP_LIST_PER_PAGE + }), + mcpViewer(workspace) + ]) return resources - .filter((r) => enabled.includes(r.path)) + .filter( + (r) => enabled.includes(r.path) && isOwnOrSharedMcpPath(r.path, r.extra_perms, viewer) + ) .map((r) => ({ path: r.path, editedAt: r.edited_at })) } catch (e) { console.error('Failed to load MCP servers', e) diff --git a/frontend/src/lib/components/mcp/mcpMenu.svelte.ts b/frontend/src/lib/components/mcp/mcpMenu.svelte.ts index aea4a7b114..a911d71c07 100644 --- a/frontend/src/lib/components/mcp/mcpMenu.svelte.ts +++ b/frontend/src/lib/components/mcp/mcpMenu.svelte.ts @@ -8,6 +8,7 @@ import type { Item } from '$lib/utils' import type { AIChatManager } from '../copilot/chat/AIChatManager.svelte' import { isMcpEnabled, setMcpEnabled } from './enabledServers' import { cachedProviderKey, rememberProviderKey } from './iconCache' +import { isOwnOrSharedMcpPath, MCP_LIST_PER_PAGE, mcpViewer } from './ownServers' import { loadProviderIcon } from './providerIcon' import McpServerIcon from './McpServerIcon.svelte' @@ -56,17 +57,22 @@ export class McpMenu { async #load(ws: string) { const seq = ++this.#seq try { - const resources = await ResourceService.listResource({ - workspace: ws, - resourceType: 'mcp', - perPage: 100 - }) + const [resources, viewer] = await Promise.all([ + ResourceService.listResource({ + workspace: ws, + resourceType: 'mcp', + perPage: MCP_LIST_PER_PAGE + }), + mcpViewer(ws) + ]) if (seq !== this.#seq) return - this.#rows = resources.map((r) => ({ - path: r.path, - editedAt: r.edited_at, - enabled: isMcpEnabled(ws, r.path) - })) + this.#rows = resources + .filter((r) => isOwnOrSharedMcpPath(r.path, r.extra_perms, viewer)) + .map((r) => ({ + path: r.path, + editedAt: r.edited_at, + enabled: isMcpEnabled(ws, r.path) + })) this.#rowsWorkspace = ws void this.#loadIcons(ws, seq) } catch { diff --git a/frontend/src/lib/components/mcp/ownServers.test.ts b/frontend/src/lib/components/mcp/ownServers.test.ts new file mode 100644 index 0000000000..d6982790ba --- /dev/null +++ b/frontend/src/lib/components/mcp/ownServers.test.ts @@ -0,0 +1,53 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { session, roleMock } = vi.hoisted(() => ({ + session: { workspace_id: 'browsed', username: 'alice', pgroups: ['g/browsed-only'] }, + roleMock: vi.fn() +})) + +vi.mock('$lib/stores', () => ({ + userStore: { subscribe: (run: (v: unknown) => void) => (run({ ...session }), () => {}) } +})) +vi.mock('$lib/user', () => ({ getWorkspaceRole: roleMock })) + +import { isOwnOrSharedMcpPath, mcpViewer } from './ownServers' + +const alice = { username: 'alice', pgroups: ['g/eng'] } + +describe('isOwnOrSharedMcpPath', () => { + it('keeps own, folder, and explicitly shared servers; drops the rest of u/', () => { + expect(isOwnOrSharedMcpPath('u/alice/notion', {}, alice)).toBe(true) + expect(isOwnOrSharedMcpPath('f/team/notion', {}, alice)).toBe(true) + expect(isOwnOrSharedMcpPath('u/bob/notion', {}, alice)).toBe(false) + expect(isOwnOrSharedMcpPath('u/alicex/notion', {}, alice)).toBe(false) + expect(isOwnOrSharedMcpPath('u/bob/notion', { 'u/alice': false }, alice)).toBe(true) + expect(isOwnOrSharedMcpPath('u/bob/notion', { 'g/eng': false }, alice)).toBe(true) + expect(isOwnOrSharedMcpPath('u/bob/notion', { 'g/ops': true }, alice)).toBe(false) + expect(isOwnOrSharedMcpPath('u/alice/notion', {}, { username: undefined, pgroups: [] })).toBe( + false + ) + }) +}) + +describe('mcpViewer', () => { + beforeEach(() => roleMock.mockReset()) + + it('answers from the store only for the browsed workspace', async () => { + expect(await mcpViewer('browsed')).toEqual({ username: 'alice', pgroups: ['g/browsed-only'] }) + expect(roleMock).not.toHaveBeenCalled() + }) + + it('looks the identity up for another workspace instead of reusing the browsed one', async () => { + roleMock.mockResolvedValue({ + kind: 'resolved', + user: { username: 'alice_other', pgroups: ['g/eng'] } + }) + expect(await mcpViewer('other')).toEqual({ username: 'alice_other', pgroups: ['g/eng'] }) + expect(roleMock).toHaveBeenCalledWith('other') + }) + + it('yields no identity when the lookup fails', async () => { + roleMock.mockResolvedValue({ kind: 'lookup_failed' }) + expect(await mcpViewer('other')).toEqual({ username: undefined, pgroups: [] }) + }) +}) diff --git a/frontend/src/lib/components/mcp/ownServers.ts b/frontend/src/lib/components/mcp/ownServers.ts new file mode 100644 index 0000000000..31c860a039 --- /dev/null +++ b/frontend/src/lib/components/mcp/ownServers.ts @@ -0,0 +1,48 @@ +import { get } from 'svelte/store' +import { userStore } from '$lib/stores' +import { getWorkspaceRole } from '$lib/user' + +/** The principals an `extra_perms` entry can name to grant this user access. */ +export type McpViewer = { username: string | undefined; pgroups: string[] } + +/** Wide enough that a workspace's whole MCP catalog arrives at once: the filter + * below runs after the server's LIMIT, so a short page could drop the viewer's + * own rows while foreign `u/` rows fill it. */ +export const MCP_LIST_PER_PAGE = 1000 + +/** + * Whether the chat lists this MCP server. A server under another user's `u/` + * prefix carries that person's credentials, and an admin's database role lets + * the resource listing return it. Acting through it would call the provider + * as them, so the chat only offers it when its owner shared it explicitly. + */ +export function isOwnOrSharedMcpPath( + path: string, + extraPerms: Record | undefined, + viewer: McpViewer +): boolean { + if (!path.startsWith('u/')) return true + if (viewer.username !== undefined && path.startsWith(`u/${viewer.username}/`)) return true + if (!extraPerms) return false + return ( + (viewer.username !== undefined && `u/${viewer.username}` in extraPerms) || + viewer.pgroups.some((g) => g in extraPerms) + ) +} + +/** + * The viewer's identity in `workspace`. Username and groups are per workspace, + * and a session chat can operate on a workspace other than the one being + * browsed, so `userStore` only answers when it describes that same workspace. + * A failed lookup yields no username: every `u/` server is then hidden rather + * than judged against another workspace's identity. + */ +export async function mcpViewer(workspace: string): Promise { + const u = get(userStore) + if (u?.workspace_id === workspace) return { username: u.username, pgroups: u.pgroups ?? [] } + const role = await getWorkspaceRole(workspace) + if (role.kind === 'resolved') { + return { username: role.user.username, pgroups: role.user.pgroups ?? [] } + } + return { username: undefined, pgroups: [] } +} From 56e21bce832182528562688acd13ec416e01ebfc Mon Sep 17 00:00:00 2001 From: Davide Modolo <36373601+davidemodolo@users.noreply.github.com> Date: Mon, 14 Sep 2026 19:05:45 +0200 Subject: [PATCH 03/44] fix(flows): stop re-evaluating skip_if once a loop is in progress (#11008) * fix(flows): stop re-evaluating skip_if once a loop is in progress skip_if is a one-time entry gate, but the flow stays at the same step for a loop's whole lifetime, so it gets re-evaluated on every iteration. previous_id stays pinned to the module preceding the loop, but once the loop is InProgress the last completed job is an inner iteration, and the results proxy in windmill-jseval aliases results. to that job's result. skip_if then reads the wrong value and can flip the loop's module to skipped after one iteration. Skip the check once status_module is already InProgress. * fix(flows): match skip_if gate to sibling entry-state allowlists Rewrite the skip_if gate as a positive allowlist (WaitingForPriorSteps | WaitingForEvents | WaitingForExecutor), matching the shape already used by the BranchOne/BranchAll predicate gates, instead of a negative filter on InProgress. Restart-at-iteration also enters as InProgress; document it as a separate case rather than folding it into the aliasing reason, which does not apply there. Add a regression test pinning skip_if to run once at while-loop entry. --- backend/tests/worker.rs | 77 ++++++++++++++++++++++ backend/windmill-worker/src/worker_flow.rs | 13 +++- 2 files changed, 89 insertions(+), 1 deletion(-) diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index 3e2bae5092..b8bd6b30a7 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -5649,6 +5649,83 @@ async fn test_whileloop_propagates_inner_iterator_eval_failure( Ok(()) } +#[cfg(all(feature = "quickjs", feature = "python"))] +#[sqlx::test(fixtures("base"))] +async fn test_whileloop_skip_if_evaluated_once_at_entry(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + // Regression test for #11007: `skip_if` on a while-loop module must be + // evaluated once, at loop entry, using the preceding step's result. + // Re-evaluating it on every iteration aliases `results.first` to the + // previous iteration's own result instead, which here lacks `.ok` and + // makes `skip_if` incorrectly turn true after the first iteration. + let port = 123; + let flow: FlowValue = serde_json::from_value(serde_json::json!({ + "modules": [ + { + "id": "first", + "value": { + "type": "rawscript", + "language": "python3", + "content": "def main(): return {\"ok\": True}", + }, + }, + { + "id": "outer", + "value": { + "type": "whileloopflow", + "skip_failures": false, + "modules": [ + { + "id": "inner", + "value": { + "input_transforms": { + "i": { + "type": "javascript", + "expr": "flow_input.iter.index", + }, + }, + "type": "rawscript", + "language": "python3", + "content": "def main(i): return i", + }, + }, + ], + }, + "skip_if": { "expr": "!results.first.ok" }, + "stop_after_if": { + "expr": "result >= 2", + "skip_if_stopped": false, + }, + }, + ], + })) + .unwrap(); + let job = JobPayload::RawFlow { value: flow, path: None, restarted_from: None }; + + let cjob = RunJob::from(job).run_until_complete(&db, false, port).await; + + assert!(cjob.success, "flow should succeed"); + + let outer_module = get_module(&cjob, "outer").expect("outer module status"); + match outer_module { + windmill_common::flow_status::FlowStatusModule::Success { skipped, flow_jobs, .. } => { + assert!( + !skipped, + "while-loop must not be skipped: skip_if should only run once, at entry" + ); + assert_eq!( + flow_jobs.map(|v| v.len()), + Some(3), + "while-loop should run 3 iterations before stop_after_if halts it" + ); + } + other => panic!("expected outer module to be Success, got {other:?}"), + } + + Ok(()) +} + #[cfg(all(feature = "quickjs", feature = "python"))] #[sqlx::test(fixtures("base"))] async fn test_stop_after_all_iters_if_bad_expr_parallel_branchall( diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index 31dddcaf5f..247e618dab 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -3891,7 +3891,18 @@ async fn push_next_flow_job( drop(resume_messages); - let is_skipped = if let Some(skip_if) = &module.skip_if { + // `skip_if` is a one-time entry gate, so only first-entry statuses evaluate it. + // Once the module is looping, the last completed job is an inner iteration, not + // `previous_id`'s, and re-evaluating would alias `results.` to it. + // A restart-at-iteration also enters as `InProgress`: it resumes without re-gating. + let is_skipped = if let Some(skip_if) = module.skip_if.as_ref().filter(|_| { + matches!( + status_module, + FlowStatusModule::WaitingForPriorSteps { .. } + | FlowStatusModule::WaitingForEvents { .. } + | FlowStatusModule::WaitingForExecutor { .. } + ) + }) { let idcontext = get_transform_context(&flow_job, previous_id.as_str(), &status); let skip_if_res = compute_bool_from_expr( &skip_if.expr, From d54a66f15c09c34f7a2b45c2e6e9649807a19265 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Mon, 14 Sep 2026 19:07:33 +0200 Subject: [PATCH 04/44] fix(cli): say where a sync push deleted variable or resource went (#10851) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(cli): keep variables and resources a sync push repo never tracked `wmill sync push` archives a script it no longer finds locally, but hard-deletes a variable or a resource: the credentials go for good. A remote-only one is equally a deletion being deployed and one the repository never had, provisioned on the instance or written by a script at runtime, and reading the second as a deletion is unrecoverable. Committed history tells them apart. A push whose changeset deletes a variable or resource now asks what this branch has ever tracked at `*.variable.*` / `*.resource.*`; anything it has never recorded is kept on the remote (prompted for on a TTY), and a real deletion, recorded before the commit that removed it, still applies. Where the history cannot be read (shallow clone, sparse checkout, no repository) there is no evidence either way, so the deletion stands as before with a warning naming the remedy — the git-sync "Pull from repo" job runs in a depth-1 clone and must keep deploying the deletions it always has. `--delete-untracked-secrets` / `deleteUntrackedSecrets` opts a mirror-semantics pipeline back into deleting them unattended. Fixes GIT-980 Co-Authored-By: Claude Opus 5 * fix(cli): classify secret-bearing deletions the way the push itself does Three ways the suffix match missed: - A fileset child can be any file, `inner.resource.yaml` included, and its deletion re-pushes the parent rather than deleting anything. Classifying with the push's own `getTypeStrFromPath`, behind the same fileset exclusion the apply loop uses, keeps the two in step. - Deleting `f/x.resource.file.ini` deletes the resource `f/x` outright, so without that file in the pathspecs every file resource walked past the check. Its two files now count as the one resource they delete. - A `specificItems` item is committed as `y..variable.yaml` while `elementsToMap` collapses it to the base path the changeset carries, so a deletion the user did commit read as never tracked. The history is searched under both names. `gitRecordedPaths` also reads its history with `core.quotePath=false`: a path with a non-ASCII byte came back C-quoted and matched nothing. Co-Authored-By: Claude Opus 5 * fix(cli): judge held-back deletions per object, not per file `DELETE /variables/delete` takes the resource at the same path down with it, and `DELETE /resources/delete` does the same to the variables its value references, so a tracked deletion could destroy an untracked object the push had just reported it was keeping. A file resource had the same shape from the other end: two files for one resource, either survivor deleting it. The unit is the server-side object. One file left unaccounted for by history now holds the whole object back, so nothing in a group reported as kept is deleted. The residual is a resource whose value references a variable at another path, which stays possible and is called out in the PR. Also corrects what the messages claim. Deleting a variable or resource is not irrecoverable: both move to the workspace trash, which keeps them for three days (migrations/20260326000000_trashbin.up.sql, CE since v1.665.0). The asymmetry with a script is real but narrower, and the prompt defaults to No, so it should say what it actually costs. Co-Authored-By: Claude Opus 5 * fix(cli): warn when sync push deletes variables the repo never tracked `sync push` deletes a remote variable or resource that has no local file. An object the repository has never tracked was provisioned outside it, by hand or by a script at runtime, rather than deleted from it, and the change list said nothing to tell the two apart. The push still deploys every deletion — that is what the repo-is-the-mirror contract means, and a shallow clone (the git-sync "Pull from repo" job, a default actions/checkout) could not tell them apart anyway. What changes is that the preview names the ones this branch's history has no record of, before the prompt that confirms them, and points at the excludes that stop them recurring. Deleting is also not final, which the CLI was alone in not saying: both handlers move the item to the workspace trash first, restorable for three days (migrations/20260326000000_trashbin.up.sql, CE since v1.665.0). A push that deleted any now says so. This replaces the earlier hold-back design. Keeping objects back changed what a push deploys, needed a flag and a wmill.yaml key to opt out of, and could claim to keep an object that a linked deletion then cascaded onto. Reporting cannot. Co-Authored-By: Claude Opus 5 * fix(cli): name the paths the untracked-deletion warning is about A push can delete a tracked and a never-tracked resource together, where "1 resource" identified neither. The warning lists the paths instead of counting them, so the reader knows which one to exclude. Outside a git checkout it no longer opens "This branch's history", which contradicted the reason it went on to give, and it drops the pronouns that disagreed with a plural count. The history walk is skipped under --json-output, where both notices are silenced and its result had no reader. Co-Authored-By: Claude Opus 5 * fix(cli): vouch for an object with any file in history, not just deleted ones The tracked set was built from the deletions being judged, so a companion file the push was not deleting could not vouch for its object: a file resource whose `.resource.yaml` stays while its content file goes was reported as never tracked, though the repository plainly owned it. It is built from the whole history now, which also turns the workspace-specific lookup around — history is normalized to base paths, the form the changeset already carries, instead of each candidate being searched for under two names. The warning also prints one line per server-side object rather than per file, so a file resource is the one deletion it is rather than two. Co-Authored-By: Claude Opus 5 * fix(cli): name both kinds at a shared path, gate the history path conversion Three from review: A variable and a resource at one path are judged together, since deleting either takes both, but they are two objects to name — keying the printed lines by path alone dropped one of them. `fromWorkspaceSpecificPath` strips a `.` segment wherever it finds one, so a history entry that merely looks workspace-suffixed was re-keyed onto a different object, whose history then vouched for it. Only a path `specificItems` claims is converted now. Not reachable from a server object (the backend rejects `.` in paths), but history holds whatever was committed. `secretBearingKey` lost its last caller when the tracked set moved to object paths; removed. Co-Authored-By: Claude Opus 5 * fix(cli): identify a secret-bearing object by kind as well as path A variable and a resource can share a path and are still two backend objects: `DELETE /variables/delete` drops the same-path resource unconditionally, while `DELETE /resources/delete` drops only the variables its value references. Keying tracked history by path alone let a committed variable vouch for a resource the repository never had, which then went unmentioned. The cascade is a reason to report both, not to treat them as one. A file resource's two files keep one id. `git log HEAD` also fails on a repository with no commits, which was reported as "its history could not be read. Check that git runs correctly in this directory" — true of neither. Co-Authored-By: Claude Opus 5 * docs(cli): tighten the comments on the untracked-deletion warning Halves the prose without dropping a constraint: the trashbin retention is stated where the message says it rather than twice more in doc comments, and the two stacked comments at the print site had come to contradict each other, one still describing a same-path variable and resource as judged together. Co-Authored-By: Claude Opus 5 * fix(cli): say where a deleted variable or resource went `sync push` deletes a remote variable or resource that has no local file, and said nothing more. Both handlers move the item to the workspace trash first, restorable for three days (migrations/20260326000000_trashbin.up.sql, CE since v1.665.0), and the CLI was the one surface never to mention it — the report behind this concluded the deletion was final and there was nothing to restore. A push that deleted any now ends with where they went and how long they have. Drops the untracked-deletion warning this branch carried: distinguishing a deletion the repo deployed from an object it never owned needs the branch's git history, and roughly 130 lines to read it and be right about the answer, for a claim the trash already softens. Two fixes it turned up in the data-table migration guard, which reads history the same way, are kept: a path with a non-ASCII byte came back C-quoted and matched nothing, and a repository with no commits was reported as one where git does not run. Fixes GIT-980 Co-Authored-By: Claude Opus 5 * fix(cli): name the trashbin correctly and say who can restore The tab is labelled Trashbin, not Trash, and `restore_trash_item` requires admin, so a non-admin reading the old line would go looking for a control they do not have. Co-Authored-By: Claude Opus 5 * refactor(cli): move the migration-guard git fixes to their own PR They fix `gitRecordedDatatableMigrationPaths`, which this PR no longer touches. Co-Authored-By: Claude Opus 5 * fix(cli): don't count a .lock deletion the push skips The apply loop `continue`s past a non-raw-app, non-dbt `.lock` deletion before reaching the delete switch, so nothing happens on the server. The classifier did not mirror that, and `f/x.resource.file.lock` reaches it as a resource through `isFileResource` — a resource type whose format_extension is literally `lock` would have the notice announce a deletion the push never performed. A raw-app or dbt `.lock`, the two that loop does not skip, classifies as its bundle's own kind well before the file-resource check, so a suffix test is enough. Co-Authored-By: Claude Opus 5 * docs(cli): state what the classification tests protect The header described the change rather than the invariant. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Opus 5 --- cli/src/commands/sync/sync.ts | 67 +++++++++++++++++++ .../secret_bearing_deletions_unit.test.ts | 66 ++++++++++++++++++ 2 files changed, 133 insertions(+) create mode 100644 cli/test/secret_bearing_deletions_unit.test.ts diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index 2159ea2539..5f67d194fe 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -3154,6 +3154,57 @@ export function untrackedDatatableMigrationDeletions< ); } +/** + * The kind of secret-bearing object whose deletion this path is, if it is one. + * + * Classified with the push's own `getTypeStrFromPath`, so this agrees with the switch + * that does the deleting. A fileset child is excluded ahead of it: it can be any file, + * `inner.resource.yaml` included, and deleting one re-pushes the parent resource + * rather than deleting anything. + */ +export function secretBearingObjectKind( + p: string, +): "variable" | "resource" | undefined { + if (isFilesetResource(p)) return undefined; + // The apply loop `continue`s past a `.lock` deletion before reaching the switch, + // so counting one would announce a deletion the push never performs. A raw-app or + // dbt `.lock`, the two that loop does not skip, classifies as its bundle's own kind + // long before the file-resource check, so a plain suffix test is enough here. + if (p.endsWith(".lock")) return undefined; + let typ: string; + try { + typ = getTypeStrFromPath(p); + } catch { + // Not a path the push classifies, so not one it deletes. + return undefined; + } + return typ === "variable" || typ === "resource" ? typ : undefined; +} + +/** The server-side object a secret-bearing file belongs to, so a file resource's two + * files are counted (and reported) as the one resource they delete. */ +function secretBearingObjectPath(p: string): string { + const normalized = p.replaceAll(SEP, "/"); + return secretBearingObjectKind(p) === "resource" + ? removeResourceSuffix(normalized) + : normalized.replace(/\.variable\.(yaml|json)$/, ""); +} + +/** e.g. "2 variables and 1 resource", counted by object rather than by file. */ +export function describeSecretBearingChanges( + changes: { path: string }[], +): string { + const objects = { variable: new Set(), resource: new Set() }; + for (const c of changes) { + const kind = secretBearingObjectKind(c.path); + if (kind) objects[kind].add(secretBearingObjectPath(c.path)); + } + return (["variable", "resource"] as const) + .filter((k) => objects[k].size > 0) + .map((k) => `${objects[k].size} ${k}${objects[k].size > 1 ? "s" : ""}`) + .join(" and "); +} + /** * Whether a pull change removes a local dbt descriptor. A dbt project's * descriptor is optional and the remote spells "this project names none" as @@ -6548,6 +6599,22 @@ export async function push( ), ); } + // Both delete handlers move the item to the workspace trashbin first; without + // this the CLI is the only surface that never says so, and the deletion reads + // as final. + const deletedSecretBearing = changes.filter( + (c) => + c.name === "deleted" && + secretBearingObjectKind(c.path) !== undefined && + !failedChanges.some((f) => f.path === c.path), + ); + if (deletedSecretBearing.length > 0) { + log.info( + colors.gray( + `${describeSecretBearingChanges(deletedSecretBearing)} deleted. The workspace trashbin keeps a deleted item for three days; a workspace admin can restore it from Workspace settings -> Trashbin.`, + ), + ); + } if (failedChanges.length > 0) { // Not process.exit: under Node a piped stdout write is async, so exiting // here would truncate the JSON result mid-object for CI consumers. diff --git a/cli/test/secret_bearing_deletions_unit.test.ts b/cli/test/secret_bearing_deletions_unit.test.ts new file mode 100644 index 0000000000..1fe2603527 --- /dev/null +++ b/cli/test/secret_bearing_deletions_unit.test.ts @@ -0,0 +1,66 @@ +/** + * The trashbin notice a push prints is only as good as its classification, which has + * to agree with the apply loop on two things: which deleted files are a variable or a + * resource — a path the loop skips must not be counted, or the notice announces a + * deletion that never happened — and that the unit is the server-side object, so a + * file resource's two files are the one deletion they cause. + */ + +import { describe, expect, test } from "bun:test"; +import { + secretBearingObjectKind, + describeSecretBearingChanges, +} from "../src/commands/sync/sync.ts"; + +describe("secretBearingObjectKind", () => { + test("matches variable and resource metadata in both serializations", () => { + expect(secretBearingObjectKind("f/test/protocol.variable.yaml")).toBe( + "variable", + ); + expect(secretBearingObjectKind("f/test/erp_access.resource.json")).toBe( + "resource", + ); + // A file resource's content file deletes the resource outright, so it counts. + expect(secretBearingObjectKind("f/test/conf.resource.file.ini")).toBe( + "resource", + ); + }); + + test("ignores files that only look like one", () => { + for (const p of [ + "f/test/my_type.resource-type.json", + // A fileset child can be any file; deleting one re-pushes the parent resource + // rather than deleting anything. + "f/test/data.fileset/edge/inner.resource.yaml", + "f/test/bar.script.yaml", + "f/test/foo.flow/flow.yaml", + // The apply loop skips a `.lock` deletion outright, so counting one would + // announce a deletion that never happens. Reachable for a resource type whose + // format_extension is literally `lock`. + "f/test/conf.resource.file.lock", + ]) { + expect(secretBearingObjectKind(p)).toBeUndefined(); + } + }); +}); + +describe("describeSecretBearingChanges", () => { + test("counts each kind separately and pluralizes", () => { + expect( + describeSecretBearingChanges([ + { path: "f/a.variable.yaml" }, + { path: "f/b.variable.yaml" }, + { path: "f/c.resource.yaml" }, + ]), + ).toBe("2 variables and 1 resource"); + }); + + test("counts a file resource's two files as the one resource they delete", () => { + expect( + describeSecretBearingChanges([ + { path: "f/c.resource.yaml" }, + { path: "f/c.resource.file.ini" }, + ]), + ).toBe("1 resource"); + }); +}); From 94c548cd5dc43131e0471b0edabe496dff245862 Mon Sep 17 00:00:00 2001 From: Guilhem Date: Mon, 14 Sep 2026 19:27:27 +0200 Subject: [PATCH 05/44] fix: re-attach flow chat to the same job on SSE timeout instead of re-running it (#11122) * fix: re-attach flow chat to the same job on SSE timeout instead of re-running it Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01S4xKKyrBTs35MEScYucgZW * fix: restart the flow chat stream when the streaming sub-job changes across a reconnect Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01S4xKKyrBTs35MEScYucgZW --------- Co-authored-by: Claude Fable 5.1 --- .../conversations/FlowChatManager.svelte.ts | 341 ++++++++++-------- 1 file changed, 185 insertions(+), 156 deletions(-) diff --git a/frontend/src/lib/components/flows/conversations/FlowChatManager.svelte.ts b/frontend/src/lib/components/flows/conversations/FlowChatManager.svelte.ts index 7cf2c8d6f7..bc8033e45d 100644 --- a/frontend/src/lib/components/flows/conversations/FlowChatManager.svelte.ts +++ b/frontend/src/lib/components/flows/conversations/FlowChatManager.svelte.ts @@ -18,6 +18,17 @@ export interface ConversationWithDraft extends FlowConversation { isDraft?: boolean } +// Per-turn stream state, kept across SSE reconnects to the same job. +interface StreamTurnState { + accumulatedContent: string + assistantMessageId: string + // Last offset the server reported; sent back on reconnect so the stream resumes + // after the deltas already rendered rather than replaying from the start. + // It indexes the stream of `streamJobId` only. + streamOffset: number | undefined + streamJobId: string | undefined +} + export class FlowChatManager { // State messages = $state([]) @@ -484,171 +495,22 @@ export class FlowChatManager { this.currentEventSource.close() } - // Track stream state for this message - let accumulatedContent = '' - let assistantMessageId = '' - let isCompleted = false - try { const jobId = await this.#onRunFlow?.(messageContent, currentConversationId, additionalInputs) if (!jobId) { console.error('No jobId returned from onRunFlow') return } + this.currentJobId = jobId - // Build the EventSource URL - const streamUrl = `/api/w/${this.#workspace()}/jobs_u/getupdate_sse/${jobId}` - const url = new URL(streamUrl, window.location.origin) - url.searchParams.set('poll_delay_ms', '50') - url.searchParams.set('fast', 'true') - url.searchParams.set('only_result', 'true') - // Create EventSource connection - const eventSource = new EventSource(url.toString()) - this.currentEventSource = eventSource - - // start polling this.startPolling(currentConversationId, isNewConversation) - eventSource.onmessage = async (event) => { - try { - const data = JSON.parse(event.data) - const type = data.type - - // Handle timeout - reconnect to SSE - if (type === 'timeout') { - eventSource.close() - this.currentEventSource = undefined - // Reconnect - this.handleStreamingMessage( - messageContent, - currentConversationId, - isNewConversation, - additionalInputs - ) - return - } - - // Handle ping - just ignore - if (type === 'ping') { - return - } - - // Handle error - if (type === 'error') { - eventSource.close() - this.currentEventSource = undefined - console.error('SSE error:', data) - sendUserToast('Stream error: ' + (data.error || 'Unknown error'), true) - this.cleanup() - return - } - - // Handle not found - if (type === 'not_found') { - eventSource.close() - this.currentEventSource = undefined - console.error('Job not found') - sendUserToast('Job not found', true) - this.cleanup() - return - } - - if (type === 'update') { - if (data.flow_stream_job_id) { - this.currentJobId = data.flow_stream_job_id - } - // Process new stream content - if (data.new_result_stream) { - // Stop polling since we are receiving last step streaming - this.stopPolling() - const { - type, - content: newContent, - success - } = parseStreamDeltas(data.new_result_stream) - accumulatedContent += newContent - - // Create tool message if type is tool_result - if (type === 'tool_result') { - // set last message streaming to false - this.messages = this.messages.map((msg) => - msg.id === this.messages[this.messages.length - 1].id - ? { ...msg, streaming: false } - : msg - ) - - this.messages = [ - ...this.messages, - { - id: 'temp-' + randomUUID(), - content: newContent, - created_at: new Date().toISOString(), - created_seq: 0, - message_type: 'tool', - conversation_id: currentConversationId, - job_id: '', - loading: false, - streaming: false, - success - } - ] - // Reset assistant message ID since we are creating a tool message - assistantMessageId = '' - accumulatedContent = '' - } - - // Create message on first content - else if ( - type === 'message' && - assistantMessageId.length === 0 && - accumulatedContent.length > 0 - ) { - assistantMessageId = 'temp-' + randomUUID() - this.messages = [ - ...this.messages, - { - id: assistantMessageId, - content: accumulatedContent, - created_at: new Date().toISOString(), - created_seq: 0, - message_type: 'assistant', - conversation_id: currentConversationId, - job_id: '', - loading: false, - streaming: true - } - ] - } else { - // Update existing message - this.messages = this.messages.map((msg) => - msg.id === assistantMessageId ? { ...msg, content: accumulatedContent } : msg - ) - } - } - - // Handle completion - if (data.completed) { - isCompleted = true - // Do a final poll to get all messages from database - if (this.selectedConversationId) { - await this.pollConversationMessages(this.selectedConversationId, { - removeTempMessages: true - }) - } - this.cleanup() - } - } - } catch (error) { - console.error('Error processing stream event:', error) - } - } - - eventSource.onerror = (error) => { - if (isCompleted) return - console.error('EventSource error:', error) - sendUserToast('Stream error occurred', true) - this.cleanup() - } + this.#followJob(jobId, currentConversationId, { + accumulatedContent: '', + assistantMessageId: '', + streamOffset: undefined, + streamJobId: undefined + }) } catch (error) { console.error('Stream connection error:', error) sendUserToast('Failed to connect to stream', true) @@ -656,6 +518,173 @@ export class FlowChatManager { } } + // Opens an SSE connection on an already-running job. The server closes every + // stream after TIMEOUT_SSE_STREAM, so a timeout re-enters here with the same + // job and turn state rather than starting a new run. + #followJob(jobId: string, currentConversationId: string, turn: StreamTurnState) { + const streamUrl = `/api/w/${this.#workspace()}/jobs_u/getupdate_sse/${jobId}` + const url = new URL(streamUrl, window.location.origin) + url.searchParams.set('poll_delay_ms', '50') + url.searchParams.set('fast', 'true') + url.searchParams.set('only_result', 'true') + if (turn.streamOffset !== undefined) { + url.searchParams.set('stream_offset', turn.streamOffset.toString()) + } + const eventSource = new EventSource(url.toString()) + this.currentEventSource = eventSource + let isCompleted = false + + eventSource.onmessage = async (event) => { + try { + const data = JSON.parse(event.data) + const type = data.type + + if (type === 'timeout') { + eventSource.close() + this.currentEventSource = undefined + this.#followJob(jobId, currentConversationId, turn) + return + } + + // Handle ping - just ignore + if (type === 'ping') { + return + } + + // Handle error + if (type === 'error') { + eventSource.close() + this.currentEventSource = undefined + console.error('SSE error:', data) + sendUserToast('Stream error: ' + (data.error || 'Unknown error'), true) + this.cleanup() + return + } + + // Handle not found + if (type === 'not_found') { + eventSource.close() + this.currentEventSource = undefined + console.error('Job not found') + sendUserToast('Job not found', true) + this.cleanup() + return + } + + if (type === 'update') { + if (data.flow_stream_job_id) { + this.currentJobId = data.flow_stream_job_id + if (data.flow_stream_job_id !== turn.streamJobId) { + const offsetFromOtherJob = + turn.streamJobId !== undefined && turn.streamOffset !== undefined + turn.streamJobId = data.flow_stream_job_id + if (offsetFromOtherJob) { + // The offset indexes the previous sub-job's stream (a retried last step + // gets a new one), so this connection skipped the new job's first chunks. + // Drop this delta and re-attach from the start of the new sub-job. + turn.streamOffset = undefined + eventSource.close() + this.currentEventSource = undefined + this.#followJob(jobId, currentConversationId, turn) + return + } + } + } + if (data.stream_offset !== undefined) { + turn.streamOffset = data.stream_offset + } + // Process new stream content + if (data.new_result_stream) { + // Stop polling since we are receiving last step streaming + this.stopPolling() + const { type, content: newContent, success } = parseStreamDeltas(data.new_result_stream) + turn.accumulatedContent += newContent + + // Create tool message if type is tool_result + if (type === 'tool_result') { + // set last message streaming to false + this.messages = this.messages.map((msg) => + msg.id === this.messages[this.messages.length - 1].id + ? { ...msg, streaming: false } + : msg + ) + + this.messages = [ + ...this.messages, + { + id: 'temp-' + randomUUID(), + content: newContent, + created_at: new Date().toISOString(), + created_seq: 0, + message_type: 'tool', + conversation_id: currentConversationId, + job_id: '', + loading: false, + streaming: false, + success + } + ] + // Reset assistant message ID since we are creating a tool message + turn.assistantMessageId = '' + turn.accumulatedContent = '' + } + + // Create message on first content + else if ( + type === 'message' && + turn.assistantMessageId.length === 0 && + turn.accumulatedContent.length > 0 + ) { + turn.assistantMessageId = 'temp-' + randomUUID() + this.messages = [ + ...this.messages, + { + id: turn.assistantMessageId, + content: turn.accumulatedContent, + created_at: new Date().toISOString(), + created_seq: 0, + message_type: 'assistant', + conversation_id: currentConversationId, + job_id: '', + loading: false, + streaming: true + } + ] + } else { + // Update existing message + this.messages = this.messages.map((msg) => + msg.id === turn.assistantMessageId + ? { ...msg, content: turn.accumulatedContent } + : msg + ) + } + } + + // Handle completion + if (data.completed) { + isCompleted = true + // Do a final poll to get all messages from database + if (this.selectedConversationId) { + await this.pollConversationMessages(this.selectedConversationId, { + removeTempMessages: true + }) + } + this.cleanup() + } + } + } catch (error) { + console.error('Error processing stream event:', error) + } + } + + eventSource.onerror = (error) => { + if (isCompleted) return + console.error('EventSource error:', error) + sendUserToast('Stream error occurred', true) + this.cleanup() + } + } + private async handlePollingMessage( messageContent: string, currentConversationId: string, From 5d32b6106788b665e4752fcac63ff1ef457d604e Mon Sep 17 00:00:00 2001 From: AlexRV12 <71396855+AlexRV12@users.noreply.github.com> Date: Mon, 14 Sep 2026 19:28:09 +0200 Subject: [PATCH 06/44] feat: run a flow step test through the chat's argument form (#11114) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit test_run_step was the last run tool still starting a job on whatever the model sent. Route it through runThroughForm, as test_run_script, run_script and test_run_flow already are. A step's arguments are its own, not the flow's: it is normally fed by its input transforms, so the form is built from the step's target rather than the flow's schema. loadSchemaFromModule resolves script and subflow steps against the deployed version, which would offer the fields of code this path is not about to run, so the schema comes from the same read the job uses — inferred from a rawscript body, the draft script's content, or the subflow's own schema. The preprocessor's _ENTRYPOINT_OVERRIDE is declared by no schema, so it is added inside the resolved startJob: proposed into the form instead, the argument conforming would drop it and the preprocessor would silently run its main. Its schema is inferred rather than read off the target for the same reason a stored one cannot describe it: a schema speaks for the one entrypoint it was inferred from. executeFlowStepTestRun splits into resolveFlowStepRun plus a thin wrapper, so the flow editor's own test_run_step keeps its behaviour but for one fix it inherits: a deployed subflow step now runs with skipPreprocessor. The flow editor's step test passes it too, and a parent flow pushes a subflow step the same way (apply_preprocessor: false) — a preprocessor would take the subflow's own inputs for a trigger event. Claude-Session: https://claude.ai/code/session_018PBB2gw8FK4YmGPxu5Drbn Co-authored-by: Claude Opus 5 (1M context) --- .../copilot/chat/global/core.test.ts | 221 +++++++++++++++++- .../components/copilot/chat/global/core.ts | 138 ++++++++--- .../src/lib/components/copilot/chat/shared.ts | 186 ++++++++------- 3 files changed, 432 insertions(+), 113 deletions(-) diff --git a/frontend/src/lib/components/copilot/chat/global/core.test.ts b/frontend/src/lib/components/copilot/chat/global/core.test.ts index 5b083fd9f7..051304cafa 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.test.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.test.ts @@ -327,6 +327,7 @@ vi.mock('$lib/infer', async () => ({ inferArgs: vi.fn(async () => {}) })) +import { inferArgs } from '$lib/infer' import { buildRunsFilterSearchbarSchema } from '$lib/components/runs/runsFilter' import { buildOpenPageUrl, @@ -386,6 +387,16 @@ function getBackendDraft(kind: string, path: string, _opts?: unknown): return backendDrafts.get(`${kind}:${path}`) as V | undefined } +// inferArgs is stubbed module-wide (no wasm parser here), so a test whose form is built by +// inference has to say what the next call finds. Once, so a test that also calls write_script +// — which infers to fill the draft's schema — queues this after that write, not before. +function stubInferredProperties(properties: Record): void { + vi.mocked(inferArgs).mockImplementationOnce(async (_lang, _code, schema) => { + schema.properties = properties + return null + }) +} + const toolCallbacks: ToolCallbacks = { setToolStatus: vi.fn(), removeToolStatus: vi.fn(), @@ -5183,6 +5194,9 @@ describe('global AI tools', () => { it('test_run_step previews rawscript steps from the draft flow', async () => { const content = 'export async function main(name: string) {\n\treturn name.toUpperCase()\n}' + // The form offers the fields the step's own code declares, so the step needs a schema + // for `name` to survive it. + stubInferredProperties({ name: { type: 'string' } }) await callGlobalTool('write_flow', { path: 'f/flows/rawscript-step', summary: 'Flow with rawscript', @@ -5240,6 +5254,9 @@ describe('global AI tools', () => { ]) }) + // A script draft carries no schema, so the form infers from the draft content — the + // version about to run. + stubInferredProperties({ name: { type: 'string' } }) await withCompletedTestJob(() => callGlobalTool('test_run_step', { path: 'f/flows/script-step', @@ -5265,7 +5282,10 @@ describe('global AI tools', () => { await callGlobalTool('write_flow', { path: 'f/flows/nested-draft', summary: 'Nested draft flow', - modules: JSON.stringify(nestedModules) + modules: JSON.stringify(nestedModules), + // A subflow step's form is the subflow's own inputs, so `name` needs declaring here + // for it to survive the form. + schema: JSON.stringify(FLOW_NAME_SCHEMA) }) await callGlobalTool('write_flow', { path: 'f/flows/parent-flow', @@ -5301,6 +5321,205 @@ describe('global AI tools', () => { }) }) + it('test_run_step runs a deployed subflow step past its preprocessor', async () => { + vi.mocked(FlowService.getFlowByPath).mockResolvedValueOnce({ + path: 'f/flows/deployed-sub', + summary: 'Deployed subflow', + value: { modules: [{ id: 'sub_start', value: { type: 'identity' } }] }, + schema: FLOW_NAME_SCHEMA + } as any) + await callGlobalTool('write_flow', { + path: 'f/flows/parent-of-deployed', + summary: 'Parent flow', + modules: JSON.stringify([ + { + id: 'call_deployed', + value: { type: 'flow', path: 'f/flows/deployed-sub', input_transforms: {} } + } + ]) + }) + + await withCompletedTestJob(() => + callGlobalTool('test_run_step', { + path: 'f/flows/parent-of-deployed', + stepId: 'call_deployed', + args: { name: 'Ada' } + }) + ) + + expect(JobService.runFlowPreview).not.toHaveBeenCalled() + expect(JobService.runFlowByPath).toHaveBeenCalledWith({ + workspace: WORKSPACE, + path: 'f/flows/deployed-sub', + requestBody: { name: 'Ada' }, + skipPreprocessor: true + }) + }) + + // A step is fed by its input transforms, so its arguments are its own and the flow's + // schema describes a different set entirely. Opening the form on the flow's would offer + // fields this job ignores and drop the ones it takes. + it('test_run_step opens the form on the step, not on the flow', async () => { + const content = 'export async function main(name: string) {\n\treturn name.toUpperCase()\n}' + await callGlobalTool('write_flow', { + path: 'f/flows/step-form', + summary: 'Step form flow', + // The flow takes `customer`; the step takes `name`. Nothing links the two. + schema: JSON.stringify({ type: 'object', properties: { customer: { type: 'string' } } }), + modules: JSON.stringify([ + { + id: 'format_name', + value: { type: 'rawscript', language: 'bun', content, input_transforms: {} } + } + ]) + }) + + stubInferredProperties({ name: { type: 'string' } }) + let form: any + await withCompletedTestJob(() => + callGlobalTool( + 'test_run_step', + { + path: 'f/flows/step-form', + stepId: 'format_name', + args: { name: 'Ada', customer: 'acme' } + }, + { + ...toolCallbacks, + requestRunArgs: async (_toolId, f) => { + form = f + return { name: 'Grace' } + } + } + ) + ) + + expect(form.schema.properties).toEqual({ name: { type: 'string' } }) + expect(form.runnableKind).toBe('script') + expect(form.summary).toBe('step "format_name"') + // `customer` is the flow's argument, so the step's form never offered it. + expect(form.args).toEqual({ name: 'Ada' }) + expect(JobService.runScriptPreview).toHaveBeenCalledWith({ + workspace: WORKSPACE, + requestBody: { content, language: 'bun', args: { name: 'Grace' } } + }) + }) + + // The step runs the draft script's content, so a form built from the deployed schema + // would offer the arguments of code that is not the code about to run. + it('test_run_step opens a script step on the draft schema, not the deployed one', async () => { + const content = 'export async function main(name: string) {\n\treturn `draft ${name}`\n}' + seedBackendDraft('script', 'f/scripts/drifted', { + path: 'f/scripts/drifted', + summary: 'Drifted', + content, + language: 'bun' + }) + await callGlobalTool('write_flow', { + path: 'f/flows/drifted-step', + summary: 'Drifted step flow', + modules: JSON.stringify([ + { + id: 'call_script', + value: { type: 'script', path: 'f/scripts/drifted', input_transforms: {} } + } + ]) + }) + + // Inferred from the draft's content. Never fetching the deployed script is the point: + // its stored schema describes code this run is not about to execute. + stubInferredProperties({ name: { type: 'string' } }) + let form: any + await withCompletedTestJob(() => + callGlobalTool( + 'test_run_step', + { path: 'f/flows/drifted-step', stepId: 'call_script', args: { name: 'Ada' } }, + { ...toolCallbacks, requestRunArgs: async (_toolId, f) => ((form = f), f.args) } + ) + ) + + expect(ScriptService.getScriptByPath).not.toHaveBeenCalled() + expect(form.schema.properties).toEqual({ name: { type: 'string' } }) + }) + + // No parser emits `password`, so the draft's stored schema is the only thing carrying it. + // Rebuilding the form's fields from the content would offer the secret as a plain text + // box, and the literal typed into it would reach the job's arguments unminted. + it('test_run_step keeps the password marking of a drafted script step', async () => { + seedBackendDraft('script', 'f/scripts/secretful', { + path: 'f/scripts/secretful', + summary: 'Secretful', + content: 'export async function main(token: string) {\n\treturn 1\n}', + language: 'bun', + schema: { + type: 'object', + properties: { token: { type: 'string', password: true } }, + required: ['token'] + } + }) + await callGlobalTool('write_flow', { + path: 'f/flows/secretful-step', + summary: 'Secretful step flow', + modules: JSON.stringify([ + { + id: 'call_secretful', + value: { type: 'script', path: 'f/scripts/secretful', input_transforms: {} } + } + ]) + }) + + let form: any + await withCompletedTestJob(() => + callGlobalTool( + 'test_run_step', + { path: 'f/flows/secretful-step', stepId: 'call_secretful', args: {} }, + { ...toolCallbacks, requestRunArgs: async (_toolId, f) => ((form = f), f.args) } + ) + ) + + expect(form.schema.properties).toMatchObject({ token: { password: true } }) + }) + + // The entrypoint override is declared by no schema, so it has to be added after the form + // rather than proposed into it — anything that conforms arguments to a schema drops it, + // and the preprocessor then silently runs its `main`. + it('test_run_step keeps the preprocessor entrypoint out of the form and on the job', async () => { + const content = 'export async function preprocessor(event: string) {\n\treturn event\n}' + await callGlobalTool('write_flow', { + path: 'f/flows/preprocessed', + summary: 'Preprocessed flow', + modules: JSON.stringify([{ id: 'start', value: { type: 'identity' } }]), + preprocessor_module: JSON.stringify({ + id: 'preprocessor', + value: { type: 'rawscript', language: 'bun', content, input_transforms: {} } + }) + }) + + vi.mocked(inferArgs).mockClear() + stubInferredProperties({ event: { type: 'string' } }) + let form: any + await withCompletedTestJob(() => + callGlobalTool( + 'test_run_step', + { path: 'f/flows/preprocessed', stepId: 'preprocessor', args: { event: 'signup' } }, + { ...toolCallbacks, requestRunArgs: async (_toolId, f) => ((form = f), f.args) } + ) + ) + + // Inferred against the preprocessor entrypoint, not `main`. + expect(vi.mocked(inferArgs).mock.calls[0][3]).toBe('preprocessor') + expect(form.schema.properties).toEqual({ event: { type: 'string' } }) + expect(form.args).toEqual({ event: 'signup' }) + expect(JobService.runScriptPreview).toHaveBeenCalledWith({ + workspace: WORKSPACE, + requestBody: { + content, + language: 'bun', + args: { _ENTRYPOINT_OVERRIDE: 'preprocessor', event: 'signup' } + } + }) + }) + // The form IS the consent, so a dismissed one must leave the script unrun. it('run_script starts no job when the user cancels the form', async () => { vi.mocked(ScriptService.getScriptByPath).mockResolvedValueOnce({ diff --git a/frontend/src/lib/components/copilot/chat/global/core.ts b/frontend/src/lib/components/copilot/chat/global/core.ts index 9d459bcf71..52f4ee39ef 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.ts @@ -125,10 +125,11 @@ import { createToolDef, droppedOptionKeys, createSearchHubScriptsTool, - executeFlowStepTestRun, executeTestRun, findAndReplace, isHubPath, + resolveFlowStepRun, + SPECIAL_MODULE_IDS, type CreatedResourceTriggerKind, type PreviewCardKind, type RunFormDisplay, @@ -961,7 +962,7 @@ const testRunStepSchema = z.object({ const testRunStepToolDef = createToolDef( testRunStepSchema, 'test_run_step', - 'Execute a test run of one step in a flow by path, preferring draft flow/script content when it exists.', + "Execute a test run of one step in a flow by path, preferring draft flow/script content when it exists. `args` are the step's OWN inputs, not the flow's: a step is normally fed by its input transforms, so send what that step's code takes, not what the flow takes. The user gets an argument form prefilled with `args` and may edit or dismiss it before it runs, so fill in every argument you can infer. For a secret argument prefer `$var:` naming an existing workspace variable; a literal is minted into a short-lived secret before the run, but stays in this call.", { strict: false } ) @@ -1365,7 +1366,7 @@ ${pipelineBullet} : ' Pass items (":" entries naming the items you changed) so the review is scoped to them — omitting items preselects every pending change in the workspace' }, or mode ("draft" or "fork") to force which comparison is shown. Prefer offering this review page over calling deploy_workspace_item directly when several items changed. - For a Windmill operation no other tool covers (workers, queue state, a run's args, ...), use search_api_endpoints to find a REST endpoint, then call_api_get for reads or call_api_endpoint for mutations (the user is asked to confirm those). Always prefer a dedicated tool when one exists; endpoints for authoring or deleting scripts, flows, apps, schedules, resources, or variables are not available through the API catalog tools — use the draft tools and delete_workspace_item instead. -- Default to test_run_script, test_run_flow, or test_run_step for any run request, an existing script included; they prefer drafts and need no deployment. Use run_script or run_flow only when the user names the deployed version ("the deployed X", "in production", "for real") — a bare "run X" is not that. For those two, read the item with read_workspace_item version: "deployed" first so the arguments match the deployed schema. test_run_script, test_run_flow, run_script and run_flow all show the user an argument form prefilled with what you sent, so fill in every argument you can infer rather than asking for it in chat. +- Default to test_run_script, test_run_flow, or test_run_step for any run request, an existing script included; they prefer drafts and need no deployment. Use run_script or run_flow only when the user names the deployed version ("the deployed X", "in production", "for real") — a bare "run X" is not that. For those two, read the item with read_workspace_item version: "deployed" first so the arguments match the deployed schema. test_run_script, test_run_flow, test_run_step, run_script and run_flow all show the user an argument form prefilled with what you sent, so fill in every argument you can infer rather than asking for it in chat. test_run_step's form is the step's own inputs, not the flow's. - When a required decision is ambiguous, use askUserQuestion with two to ten clear proposed answer strings instead of guessing. The user can also type a custom answer when none of the proposed answers fit. Set multiSelect: true only when the answers can genuinely co-apply and the user may pick several (not mutually exclusive). - When the user asks you to remember a lasting preference, always/never do something, or change/stop a behavior going forward, call update_user_instructions to persist it. It edits only the USER INSTRUCTIONS block (not WORKSPACE INSTRUCTIONS). Keep each instruction concise; do not use it for one-off requests scoped to the current task. - Keep context targeted.${ @@ -3739,9 +3740,10 @@ export const globalTools: Tool<{}>[] = [ const parsed = testRunStepSchema.parse(ctx.args) return testRunFlowStepByPath(parsed, ctx) }, - requiresConfirmation: true, - confirmationMessage: (args) => - `Run a test of step "${args?.stepId ?? ''}" in ${pathLeaf(args?.path, 'the flow')}`, + // No requiresConfirmation, for the reason test_run_script carries. + bypassedByAutoAccept: true, + streamingLabel: 'Preparing the test form...', + confirmationMessage: 'Run a test of a flow step', queuedLabel: (args) => `Test step "${args?.stepId ?? ''}" of ${args?.path ?? 'the flow'}`, showDetails: true, autoCollapseDetails: false @@ -5230,19 +5232,24 @@ async function loadScriptForEdit( } /** The fields a test form offers, for code that may never have been deployed. The stored - * schema wins wherever it declares fields — a draft's or the deployed script's; only one - * declaring nothing is inferred here, from the content about to run. */ + * schema wins wherever it declares fields — a draft's or the deployed script's; one that + * declares nothing, or that speaks for an entrypoint other than the one about to run, is + * inferred here from the content instead. */ async function schemaForTestRun(script: { content: string language: ScriptLang schema?: Record + /** A preprocessor takes the arguments of its own entrypoint, not of `main`. */ + entrypoint?: 'preprocessor' }): Promise> { // Emptily declared is not declared: a stored `properties: {}` means the schema predates // the arguments the code now takes, so infer rather than offer a form with no fields. - if (Object.keys(script.schema?.properties ?? {}).length > 0) return script.schema! + // A stored schema speaks for one entrypoint, so it can never answer for an override. + if (!script.entrypoint && Object.keys(script.schema?.properties ?? {}).length > 0) + return script.schema! const schema = emptySchema() try { - await inferArgs(script.language, script.content, schema) + await inferArgs(script.language, script.content, schema, script.entrypoint) } catch (e) { console.error('Failed to infer script schema for the test run form', e) } @@ -5272,18 +5279,21 @@ async function editScript( async function loadFlowDraftValue( path: string, workspace: string -): Promise<{ flow: FlowDraftValue; summary?: string }> { + // `isDraft` says which of the two this came from. Reported here because the draft lookup + // is a request of its own: a caller that needs to know would otherwise repeat it. +): Promise<{ flow: FlowDraftValue; summary?: string; isDraft: boolean }> { const draft = await getGlobalDraft(workspace, 'flow', path) if (draft) { if (draft.value === undefined || typeof draft.value === 'string') { throw new Error(`Draft flow "${path}" has no value.`) } - return { flow: draft.value as FlowDraftValue, summary: draft.summary } + return { flow: draft.value as FlowDraftValue, summary: draft.summary, isDraft: true } } const flow = await FlowService.getFlowByPath({ workspace, path }) return { flow: { value: flow.value, schema: flow.schema, groups: flow.value.groups ?? null }, - summary: flow.summary + summary: flow.summary, + isDraft: false } } @@ -5454,30 +5464,42 @@ function flowDraftValueForPreview(flowDraft: FlowDraftValue): FlowValue { async function loadScriptForFlowStep( moduleValue: { path: string; hash?: string }, workspace: string -): Promise<{ content: string; language: ScriptLang }> { +): Promise<{ content: string; language: ScriptLang; schema?: Record }> { const draft = await getGlobalDraft(workspace, 'script', moduleValue.path) if (draft) { if (typeof draft.value !== 'string' || !draft.language) { throw new Error(`Draft script "${moduleValue.path}" is missing content or language.`) } - return { content: draft.value, language: draft.language } + return { + content: draft.value, + language: draft.language, + // The draft's own: no parser emits `password`, so a schema rebuilt from the content + // would offer a secret argument as a plain field and take the literal into the job. + schema: draft.schema as Record | undefined + } } const script = moduleValue.hash ? await ScriptService.getScriptByHash({ workspace, hash: moduleValue.hash }) : await ScriptService.getScriptByPath({ workspace, path: moduleValue.path }) - return { content: script.content, language: script.language } + return { + content: script.content, + language: script.language, + schema: script.schema as Record | undefined + } } -async function loadDraftFlowPreviewValue( +async function loadSubflowForFlowStep( path: string, workspace: string -): Promise { - if (!(await getGlobalDraft(workspace, 'flow', path))) { - return undefined - } +): Promise<{ previewValue?: FlowValue; schema?: Record }> { const nestedFlow = await loadFlowDraftValue(path, workspace) - return flowDraftValueForPreview(nestedFlow.flow) + return { + // Only a draft is previewed; a deployed subflow is run by path, as its parent flow + // would run it. The schema describes whichever of the two that leaves. + previewValue: nestedFlow.isDraft ? flowDraftValueForPreview(nestedFlow.flow) : undefined, + schema: nestedFlow.flow.schema ?? undefined + } } // Leaf of a workspace path (last segment), for human-readable confirmation @@ -5565,7 +5587,14 @@ type FormRunSpec = { toolName: string proposed: Record | null | undefined startMessage: string + /** What runs: the jobs-tray kind, and the noun the card's own prose reads. */ contextName: 'script' | 'flow' + /** What the lines the model reads back call the thing that ran. Defaults to + * `contextName`, which a flow step is not: it runs a script or a subflow but is neither. */ + noun?: string + /** What names the run where its path would not: the jobs-tray row, and the two + * background-job sentences the model reads. Those quote it, so it carries none. */ + label?: string /** Whether the bypass posture may answer this form with what it opened with. */ autoAcceptable?: boolean background?: boolean @@ -5582,8 +5611,9 @@ async function runThroughForm(spec: FormRunSpec, ctx: WriteDraftCtx): Promise { const { workspace, toolId, toolCallbacks } = ctx const flow = await loadFlowDraftValue(args.path, workspace) - const flowValue = flowDraftValueForPreview(flow.flow) - const testArgs = normalizeTestRunArgs(args.args) - - return executeFlowStepTestRun({ - flowValue, + const resolved = await resolveFlowStepRun({ + flowValue: flowDraftValueForPreview(flow.flow), stepId: args.stepId, - args: testArgs, workspace, toolCallbacks, toolId, - background: args.background, - detachAfterMs: waitSecondsToDetachMs(args.wait_seconds), loadScript: loadScriptForFlowStep, - loadFlowPreviewValue: loadDraftFlowPreviewValue + loadSubflow: loadSubflowForFlowStep }) + + // The module resolution landed on, not the id that was asked for: the job's entrypoint + // override reads the same value, and a form built for the other entrypoint offers fields + // the run will not take. + const isPreprocessor = resolved.module.id === SPECIAL_MODULE_IDS.PREPROCESSOR + // The step's own inputs, never the flow's: a step is fed by its input transforms, so the + // flow's schema names arguments this job would ignore and omits the ones it takes. + const schema = + resolved.code != undefined && resolved.lang + ? await schemaForTestRun({ + content: resolved.code, + language: resolved.lang, + schema: resolved.schema, + entrypoint: isPreprocessor ? 'preprocessor' : undefined + }) + : (resolved.schema ?? {}) + + const stepSummary = resolved.module.summary + return runThroughForm( + { + // The flow: a step has no path of its own, and this is what the status and cancel + // lines quote back, so it has to name something the reader can go and open. The + // step itself is named by the summary below. + path: args.path, + schema, + summary: stepSummary ? `step "${args.stepId}": ${stepSummary}` : `step "${args.stepId}"`, + kind: 'test', + code: resolved.code ?? schema['x-windmill-dyn-select-code'], + lang: resolved.lang ?? schema['x-windmill-dyn-select-lang'], + // Never "deployed": a step test previews the draft flow, and the step's target may + // itself be a draft. + schemaNoun: 'step', + toolName: 'test_run_step', + proposed: args.args, + startMessage: resolved.startMessage, + contextName: resolved.runnableKind, + noun: 'step', + label: `step ${args.stepId}`, + // The model is told to test and iterate, so the bypass posture answers the form. + autoAcceptable: true, + background: args.background, + detachAfterMs: waitSecondsToDetachMs(args.wait_seconds), + startJob: resolved.startJob + }, + ctx + ) } async function initApp( diff --git a/frontend/src/lib/components/copilot/chat/shared.ts b/frontend/src/lib/components/copilot/chat/shared.ts index bc00e415f6..73efd2e37e 100644 --- a/frontend/src/lib/components/copilot/chat/shared.ts +++ b/frontend/src/lib/components/copilot/chat/shared.ts @@ -2023,23 +2023,50 @@ export async function executeTestRun(config: TestRunConfig): Promise { type FlowStepScriptLoader = ( moduleValue: { path: string; hash?: string }, workspace: string -) => Promise<{ content: string; language: ScriptLang }> +) => Promise<{ content: string; language: ScriptLang; schema?: Record }> -type FlowStepPreviewLoader = (path: string, workspace: string) => Promise +/** A subflow step's target. `previewValue` is set only when a draft exists — that is what + * decides between previewing the draft and running the deployed flow by path — while + * `schema` describes whichever of the two is about to run. */ +type FlowStepSubflowLoader = ( + path: string, + workspace: string +) => Promise<{ previewValue?: FlowValue; schema?: Record } | undefined> -export type FlowStepTestRunConfig = { +type FlowStepRunConfig = { flowValue: FlowValue stepId: string - args?: Record | null workspace: string toolCallbacks: ToolCallbacks toolId: string + loadScript?: FlowStepScriptLoader + loadSubflow?: FlowStepSubflowLoader +} + +export type FlowStepTestRunConfig = FlowStepRunConfig & { + args?: Record | null background?: boolean /** Inline wait budget (ms) before the step job detaches into the tray; forwarded * to executeTestRun. Ignored when `background` is set. */ detachAfterMs?: number - loadScript?: FlowStepScriptLoader - loadFlowPreviewValue?: FlowStepPreviewLoader +} + +/** One step resolved to the job it would start, short of starting it, so a caller that + * puts an argument form in front of the run can build the form's fields from the same + * read the job uses. Schema inference lives with the caller: this module is kept on a + * shallow import list (see the note at the top of the file). */ +export type ResolvedFlowStepRun = { + module: FlowModule + runnableKind: 'script' | 'flow' + /** A subflow step carries `schema` instead, having no code of its own to read. */ + code?: string + lang?: ScriptLang + schema?: Record + startMessage: string + /** Takes the arguments as submitted. The preprocessor's entrypoint override is added + * here rather than by the caller: it is declared by no schema, so anything that + * conforms arguments to one would drop it. */ + startJob: (args: Record) => Promise } function normalizeFlowStepArgs(args: Record | null | undefined): Record { @@ -2065,25 +2092,26 @@ function getAvailableFlowStepIds(flowValue: FlowValue): string { async function loadDeployedScriptForFlowStep( moduleValue: { path: string; hash?: string }, workspace: string -): Promise<{ content: string; language: ScriptLang }> { +): Promise<{ content: string; language: ScriptLang; schema?: Record }> { const script = moduleValue.hash ? await ScriptService.getScriptByHash({ workspace, hash: moduleValue.hash }) : await ScriptService.getScriptByPath({ workspace, path: moduleValue.path }) - return { content: script.content, language: script.language } + return { + content: script.content, + language: script.language, + schema: script.schema as Record | undefined + } } -export async function executeFlowStepTestRun({ +export async function resolveFlowStepRun({ flowValue, stepId, - args, workspace, toolCallbacks, toolId, - background, - detachAfterMs, loadScript = loadDeployedScriptForFlowStep, - loadFlowPreviewValue -}: FlowStepTestRunConfig): Promise { + loadSubflow +}: FlowStepRunConfig): Promise { const targetModule = findModuleInFlow(flowValue, stepId) ?? undefined if (!targetModule) { @@ -2097,94 +2125,76 @@ export async function executeFlowStepTestRun({ } const moduleValue = targetModule.value - const stepArgs = normalizeFlowStepArgs(args) + const withEntrypoint = (args: Record) => flowStepArgsForModule(targetModule.id, args) if (moduleValue.type === 'rawscript') { - return executeTestRun({ - jobStarter: () => + return { + module: targetModule, + runnableKind: 'script', + code: moduleValue.content ?? '', + lang: moduleValue.language, + startMessage: `Starting test run of step "${stepId}"...`, + startJob: (args) => JobService.runScriptPreview({ workspace, requestBody: { content: moduleValue.content ?? '', language: moduleValue.language, - args: flowStepArgsForModule(targetModule.id, stepArgs) + args: withEntrypoint(args) } - }), - workspace, - toolCallbacks, - toolId, - startMessage: `Starting test run of step "${stepId}"...`, - contextName: 'script', - label: `step ${stepId}`, - background, - detachAfterMs - }) + }) + } } if (moduleValue.type === 'script') { const script = await loadScript(moduleValue, workspace) - return executeTestRun({ - jobStarter: () => + return { + module: targetModule, + runnableKind: 'script', + code: script.content, + lang: script.language, + schema: script.schema, + startMessage: `Starting test run of script step "${stepId}"...`, + startJob: (args) => JobService.runScriptPreview({ workspace, requestBody: { path: moduleValue.path, content: script.content, language: script.language, - args: flowStepArgsForModule(targetModule.id, stepArgs) + args: withEntrypoint(args) } - }), - workspace, - toolCallbacks, - toolId, - startMessage: `Starting test run of script step "${stepId}"...`, - contextName: 'script', - label: `step ${stepId}`, - background, - detachAfterMs - }) + }) + } } if (moduleValue.type === 'flow') { - const previewValue = await loadFlowPreviewValue?.(moduleValue.path, workspace) - if (previewValue) { - return executeTestRun({ - jobStarter: () => - JobService.runFlowPreview({ - workspace, - requestBody: { + const subflow = await loadSubflow?.(moduleValue.path, workspace) + const previewValue = subflow?.previewValue + return { + module: targetModule, + runnableKind: 'flow', + schema: subflow?.schema, + startMessage: previewValue + ? `Starting test run of draft flow step "${stepId}"...` + : `Starting test run of flow step "${stepId}"...`, + startJob: (args) => + previewValue + ? JobService.runFlowPreview({ + workspace, + requestBody: { path: moduleValue.path, value: previewValue, args } + }) + : JobService.runFlowByPath({ + workspace, path: moduleValue.path, - value: previewValue, - args: stepArgs - } - }), - workspace, - toolCallbacks, - toolId, - startMessage: `Starting test run of draft flow step "${stepId}"...`, - contextName: 'flow', - label: `step ${stepId}`, - background, - detachAfterMs - }) + requestBody: args, + // As the flow editor's own step test does: these are the subflow's main input + // schema's arguments, and a preprocessor would take them for a trigger event + // and hand the flow its own output instead. A parent flow runs a subflow step + // the same way (apply_preprocessor: false). + skipPreprocessor: true + }) } - - return executeTestRun({ - jobStarter: () => - JobService.runFlowByPath({ - workspace, - path: moduleValue.path, - requestBody: stepArgs - }), - workspace, - toolCallbacks, - toolId, - startMessage: `Starting test run of flow step "${stepId}"...`, - contextName: 'flow', - label: `step ${stepId}`, - background, - detachAfterMs - }) } toolCallbacks.setToolStatus(toolId, { @@ -2196,6 +2206,26 @@ export async function executeFlowStepTestRun({ ) } +export async function executeFlowStepTestRun({ + args, + background, + detachAfterMs, + ...config +}: FlowStepTestRunConfig): Promise { + const resolved = await resolveFlowStepRun(config) + return executeTestRun({ + jobStarter: () => resolved.startJob(normalizeFlowStepArgs(args)), + workspace: config.workspace, + toolCallbacks: config.toolCallbacks, + toolId: config.toolId, + startMessage: resolved.startMessage, + contextName: resolved.runnableKind, + label: `step ${config.stepId}`, + background, + detachAfterMs + }) +} + function formatLogs(logs: string | undefined): undefined | string { if (logs && logs.trim()) { if (logs.length <= MAX_LOG_LENGTH) { From a4ddbd733b1c4a65a6ad1621a475c3c276acd39e Mon Sep 17 00:00:00 2001 From: Guilhem Date: Mon, 14 Sep 2026 19:28:57 +0200 Subject: [PATCH 07/44] lower the workspace creation handover floor to 500ms (#11109) Claude-Session: https://claude.ai/code/session_015hBkusKsmiRiZ9VZVE314G Co-authored-by: Claude Fable 5.1 --- frontend/src/lib/workspaceCreation.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/lib/workspaceCreation.ts b/frontend/src/lib/workspaceCreation.ts index e4a9045f42..34c1644462 100644 --- a/frontend/src/lib/workspaceCreation.ts +++ b/frontend/src/lib/workspaceCreation.ts @@ -109,7 +109,7 @@ export async function enterNewWorkspace(id: string): Promise { * that time reads as nothing having happened — the floor is what makes it read as an action * that ran, and it covers the workspace layout's first load on the other side. */ -export const WORKSPACE_HANDOVER_MS = 900 +export const WORKSPACE_HANDOVER_MS = 500 /** * What to call a workspace before its owner has said. The login provider's name when it gave From 0b1e9c0dda2ae55c56b1e0de5c0419b4511b973f Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 14 Sep 2026 20:51:21 +0200 Subject: [PATCH 08/44] fix: wake a WAC parent from every path that completes its child (#11119) * fix: wake a WAC parent from every path that completes its child Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01L2ibGNBxNd8oa3uQZLHsXn * fix: park a WAC parent before writing its checkpoint so lock order matches child completion Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01L2ibGNBxNd8oa3uQZLHsXn * fix: check the parent-child link before touching a WAC parent, wrap the fallback error, keep inline checkpoints in lock order Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01L2ibGNBxNd8oa3uQZLHsXn * docs: say the zombie fallback keeps the WAC parent notification in its transaction Co-Authored-By: Claude Fable 5.1 --------- Co-authored-by: Claude Fable 5.1 --- ...2be2f0f31fb50d519b42a056d0d73417599a3.json | 16 -- ...eea41923ba4122fa666c6e8d8b06dcb5a71f.json} | 16 +- ...ed5c8c64daec45ba820a6c8f7d7ab76cc40c9.json | 28 ++ ...1e91093233cf16af0dae666b4743f3878b22e.json | 14 - ...5be9d9f51071978c0e48df4284d8b90000a4a.json | 22 -- ...933aa54523bf44d63e304440e53b9eadd5340.json | 24 -- backend/src/monitor.rs | 50 +++- .../wac_child_completion_wakes_parent.rs | 189 ++++++++++++++ backend/windmill-common/src/wac.rs | 202 +++++++++++++++ backend/windmill-queue/src/jobs.rs | 94 ++----- backend/windmill-worker/src/bun_executor.rs | 131 +++++----- .../windmill-worker/src/result_processor.rs | 245 +----------------- backend/windmill-worker/src/wac_executor.rs | 5 + backend/windmill-worker/src/worker.rs | 2 +- backend/windmill-worker/src/worker_flow.rs | 29 +-- 15 files changed, 581 insertions(+), 486 deletions(-) delete mode 100644 backend/.sqlx/query-29935e89475f637d765c516f1aa2be2f0f31fb50d519b42a056d0d73417599a3.json rename backend/.sqlx/{query-beecb176df512e4a94771d0d73c4c597e07e53d499131b57e4d6441fd0af09cb.json => query-2abc2a5830130b2b4b32983407abeea41923ba4122fa666c6e8d8b06dcb5a71f.json} (65%) create mode 100644 backend/.sqlx/query-32ca7941db013dacd2479962fa9ed5c8c64daec45ba820a6c8f7d7ab76cc40c9.json delete mode 100644 backend/.sqlx/query-a72081cb042f09034338dcb49381e91093233cf16af0dae666b4743f3878b22e.json delete mode 100644 backend/.sqlx/query-b663e6baf2f8da00c6d94e5b8e35be9d9f51071978c0e48df4284d8b90000a4a.json delete mode 100644 backend/.sqlx/query-c331609952e0b98d36f605bd5d2933aa54523bf44d63e304440e53b9eadd5340.json create mode 100644 backend/tests/wac_child_completion_wakes_parent.rs diff --git a/backend/.sqlx/query-29935e89475f637d765c516f1aa2be2f0f31fb50d519b42a056d0d73417599a3.json b/backend/.sqlx/query-29935e89475f637d765c516f1aa2be2f0f31fb50d519b42a056d0d73417599a3.json deleted file mode 100644 index e4f70e1f07..0000000000 --- a/backend/.sqlx/query-29935e89475f637d765c516f1aa2be2f0f31fb50d519b42a056d0d73417599a3.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_completed SET\n workflow_as_code_status = jsonb_set(\n jsonb_set(\n workflow_as_code_status,\n array[$1],\n COALESCE(workflow_as_code_status->$1, '{}'::jsonb)\n ),\n array[$1, 'duration_ms'],\n to_jsonb($2::bigint)\n )\n WHERE id = $3 AND workflow_as_code_status IS NOT NULL", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Int8", - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "29935e89475f637d765c516f1aa2be2f0f31fb50d519b42a056d0d73417599a3" -} diff --git a/backend/.sqlx/query-beecb176df512e4a94771d0d73c4c597e07e53d499131b57e4d6441fd0af09cb.json b/backend/.sqlx/query-2abc2a5830130b2b4b32983407abeea41923ba4122fa666c6e8d8b06dcb5a71f.json similarity index 65% rename from backend/.sqlx/query-beecb176df512e4a94771d0d73c4c597e07e53d499131b57e4d6441fd0af09cb.json rename to backend/.sqlx/query-2abc2a5830130b2b4b32983407abeea41923ba4122fa666c6e8d8b06dcb5a71f.json index 697e49ab9d..3a864fee63 100644 --- a/backend/.sqlx/query-beecb176df512e4a94771d0d73c4c597e07e53d499131b57e4d6441fd0af09cb.json +++ b/backend/.sqlx/query-2abc2a5830130b2b4b32983407abeea41923ba4122fa666c6e8d8b06dcb5a71f.json @@ -1,15 +1,23 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO v2_job_completed\n (workspace_id, id, started_at, duration_ms, result, memory_peak, status, worker)\n SELECT q.workspace_id, q.id, q.started_at,\n COALESCE((EXTRACT('epoch' FROM now()) - EXTRACT('epoch' FROM COALESCE(q.started_at, now()))) * 1000, 0)::bigint,\n $2::jsonb, r.memory_peak, 'failure'::job_status, q.worker\n FROM v2_job_queue q\n LEFT JOIN v2_job_runtime r ON r.id = q.id\n WHERE q.id = $1\n ON CONFLICT (id) DO UPDATE SET status = 'failure', result = $2::jsonb", + "query": "INSERT INTO v2_job_completed\n (workspace_id, id, started_at, duration_ms, result, memory_peak, status, worker)\n SELECT q.workspace_id, q.id, q.started_at,\n COALESCE((EXTRACT('epoch' FROM now()) - EXTRACT('epoch' FROM COALESCE(q.started_at, now()))) * 1000, 0)::bigint,\n $2::jsonb, r.memory_peak, 'failure'::job_status, q.worker\n FROM v2_job_queue q\n LEFT JOIN v2_job_runtime r ON r.id = q.id\n WHERE q.id = $1\n ON CONFLICT (id) DO UPDATE SET status = 'failure', result = $2::jsonb\n RETURNING duration_ms AS \"duration_ms!\"", "describe": { - "columns": [], + "columns": [ + { + "ordinal": 0, + "name": "duration_ms!", + "type_info": "Int8" + } + ], "parameters": { "Left": [ "Uuid", "Jsonb" ] }, - "nullable": [] + "nullable": [ + false + ] }, - "hash": "beecb176df512e4a94771d0d73c4c597e07e53d499131b57e4d6441fd0af09cb" + "hash": "2abc2a5830130b2b4b32983407abeea41923ba4122fa666c6e8d8b06dcb5a71f" } diff --git a/backend/.sqlx/query-32ca7941db013dacd2479962fa9ed5c8c64daec45ba820a6c8f7d7ab76cc40c9.json b/backend/.sqlx/query-32ca7941db013dacd2479962fa9ed5c8c64daec45ba820a6c8f7d7ab76cc40c9.json new file mode 100644 index 0000000000..0a2976f868 --- /dev/null +++ b/backend/.sqlx/query-32ca7941db013dacd2479962fa9ed5c8c64daec45ba820a6c8f7d7ab76cc40c9.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT parent_job, flow_step_id FROM v2_job WHERE id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "parent_job", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "flow_step_id", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + true, + true + ] + }, + "hash": "32ca7941db013dacd2479962fa9ed5c8c64daec45ba820a6c8f7d7ab76cc40c9" +} diff --git a/backend/.sqlx/query-a72081cb042f09034338dcb49381e91093233cf16af0dae666b4743f3878b22e.json b/backend/.sqlx/query-a72081cb042f09034338dcb49381e91093233cf16af0dae666b4743f3878b22e.json deleted file mode 100644 index 4fa871c594..0000000000 --- a/backend/.sqlx/query-a72081cb042f09034338dcb49381e91093233cf16af0dae666b4743f3878b22e.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_queue SET suspend = 0, suspend_until = NULL WHERE id = $1", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "a72081cb042f09034338dcb49381e91093233cf16af0dae666b4743f3878b22e" -} diff --git a/backend/.sqlx/query-b663e6baf2f8da00c6d94e5b8e35be9d9f51071978c0e48df4284d8b90000a4a.json b/backend/.sqlx/query-b663e6baf2f8da00c6d94e5b8e35be9d9f51071978c0e48df4284d8b90000a4a.json deleted file mode 100644 index 8d09036772..0000000000 --- a/backend/.sqlx/query-b663e6baf2f8da00c6d94e5b8e35be9d9f51071978c0e48df4284d8b90000a4a.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_queue SET suspend = GREATEST(suspend - 1, 0) WHERE id = $1 RETURNING suspend", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "suspend", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Uuid" - ] - }, - "nullable": [ - false - ] - }, - "hash": "b663e6baf2f8da00c6d94e5b8e35be9d9f51071978c0e48df4284d8b90000a4a" -} diff --git a/backend/.sqlx/query-c331609952e0b98d36f605bd5d2933aa54523bf44d63e304440e53b9eadd5340.json b/backend/.sqlx/query-c331609952e0b98d36f605bd5d2933aa54523bf44d63e304440e53b9eadd5340.json deleted file mode 100644 index efd03ae26e..0000000000 --- a/backend/.sqlx/query-c331609952e0b98d36f605bd5d2933aa54523bf44d63e304440e53b9eadd5340.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_status SET\n workflow_as_code_status = jsonb_set(\n jsonb_set(\n workflow_as_code_status,\n array[$1],\n COALESCE(workflow_as_code_status->$1, '{}'::jsonb)\n ),\n array[$1, 'duration_ms'],\n to_jsonb($2::bigint)\n )\n WHERE id = $3 AND workflow_as_code_status IS NOT NULL\n RETURNING workflow_as_code_status->'_checkpoint'->'pending_steps'->'job_ids' AS \"job_ids: serde_json::Value\"", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "job_ids: serde_json::Value", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [ - "Text", - "Int8", - "Uuid" - ] - }, - "nullable": [ - null - ] - }, - "hash": "c331609952e0b98d36f605bd5d2933aa54523bf44d63e304440e53b9eadd5340" -} diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index dc8072d684..3558155af7 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -6151,7 +6151,10 @@ async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, node_n /// Force-complete a zombie job that handle_job_error failed to complete. /// This is a minimal fallback: it inserts a failed completed job and deletes /// from the queue in a single transaction, without schedule pushing or -/// error handler logic that could cause the completion to fail. +/// error handler logic. The one thing it keeps is the WAC parent notification, +/// deliberately inside the transaction: if that fails, the whole completion +/// rolls back and the job waits for the next sweep, which is cheaper than a +/// parent parked for its full suspend window and a task run twice. async fn force_complete_zombie_job( db: &Pool, job_id: &Uuid, @@ -6173,14 +6176,18 @@ async fn force_complete_zombie_job( "Zombie job {job_id} was not completed by handle_job_error, force-completing it" ); + // Same `{"error": ...}` shape as every other failed job's result, so a WAC + // parent's failure record reads the name and message like any task failure. let error_value = serde_json::json!({ - "message": error_message, - "name": "ExecutionErr", + "error": { + "message": error_message, + "name": "ExecutionErr", + } }); let mut tx = db.begin().await?; - sqlx::query!( + let duration_ms = sqlx::query_scalar!( "INSERT INTO v2_job_completed (workspace_id, id, started_at, duration_ms, result, memory_peak, status, worker) SELECT q.workspace_id, q.id, q.started_at, @@ -6189,19 +6196,50 @@ async fn force_complete_zombie_job( FROM v2_job_queue q LEFT JOIN v2_job_runtime r ON r.id = q.id WHERE q.id = $1 - ON CONFLICT (id) DO UPDATE SET status = 'failure', result = $2::jsonb", + ON CONFLICT (id) DO UPDATE SET status = 'failure', result = $2::jsonb + RETURNING duration_ms AS \"duration_ms!\"", job_id, error_value, ) - .execute(&mut *tx) + .fetch_optional(&mut *tx) .await?; + // A WAC parent parked on this job must learn of the failure here too, or it + // waits out its whole suspend window and runs the task again. + let mut wac_parent_ready = false; + if let Some(duration_ms) = duration_ms { + let parent = sqlx::query!( + "SELECT parent_job, flow_step_id FROM v2_job WHERE id = $1", + job_id + ) + .fetch_optional(&mut *tx) + .await?; + if let Some(parent_job) = parent + .filter(|j| j.flow_step_id.is_none()) + .and_then(|j| j.parent_job) + { + wac_parent_ready = windmill_common::wac::record_child_completion( + &mut tx, + &parent_job, + job_id, + false, + duration_ms, + &error_value.to_string(), + ) + .await?; + } + } + sqlx::query!("DELETE FROM v2_job_queue WHERE id = $1", job_id) .execute(&mut *tx) .await?; tx.commit().await?; + if wac_parent_ready { + windmill_common::wac::WAC_SUSPEND_READY.store(true, Ordering::Relaxed); + } + tracing::info!("Force-completed zombie job {job_id}"); Ok(()) } diff --git a/backend/tests/wac_child_completion_wakes_parent.rs b/backend/tests/wac_child_completion_wakes_parent.rs new file mode 100644 index 0000000000..23c32573ab --- /dev/null +++ b/backend/tests/wac_child_completion_wakes_parent.rs @@ -0,0 +1,189 @@ +//! A WAC v2 parent parks on its dispatched children and is woken by their +//! completions. A child does not always complete through the worker that ran it: +//! the zombie monitor and a force cancel both go straight to +//! `add_completed_job_error`. The parent must be woken from there too, or it sits +//! out its whole suspend window and then runs the task a second time. + +use serde_json::{json, Value}; +use sqlx::{types::Json, Pool, Postgres}; +use uuid::Uuid; +use windmill_queue::{add_completed_job, add_completed_job_error, get_mini_completed_job}; + +const W_ID: &str = "test-workspace"; + +async fn insert_job(db: &Pool, id: Uuid, parent: Option) -> anyhow::Result<()> { + sqlx::query( + "INSERT INTO v2_job (id, workspace_id, created_by, created_at, permissioned_as, \ + permissioned_as_email, kind, script_lang, runnable_path, tag, visible_to_owner, parent_job) \ + VALUES ($1, $2, 'test-user', now(), 'u/test-user', 'test@windmill.dev', \ + 'script', 'bun', 'u/test-user/wac', 'bun', true, $3)", + ) + .bind(id) + .bind(W_ID) + .bind(parent) + .execute(db) + .await?; + sqlx::query( + "INSERT INTO v2_job_queue (id, workspace_id, scheduled_for, running, tag) \ + VALUES ($1, $2, now(), true, 'bun')", + ) + .bind(id) + .bind(W_ID) + .execute(db) + .await?; + Ok(()) +} + +/// A parent parked on `steps` (step key → child job), the shape +/// `handle_wac_v2_output` leaves behind once the children are pushed. +async fn plant_parked_parent(db: &Pool, steps: &[(&str, Uuid)]) -> anyhow::Result { + let parent = Uuid::new_v4(); + insert_job(db, parent, None).await?; + sqlx::query( + "UPDATE v2_job_queue SET suspend = $2, suspend_until = now() + interval '14 days' \ + WHERE id = $1", + ) + .bind(parent) + .bind(steps.len() as i32) + .execute(db) + .await?; + let job_ids: serde_json::Map = steps + .iter() + .map(|(k, id)| (k.to_string(), json!(id.to_string()))) + .collect(); + let keys: Vec<&str> = steps.iter().map(|(k, _)| *k).collect(); + sqlx::query("INSERT INTO v2_job_status (id, workflow_as_code_status) VALUES ($1, $2)") + .bind(parent) + .bind(json!({ + "_checkpoint": { + "completed_steps": {}, + "pending_steps": { "mode": "dispatch", "keys": keys, "job_ids": job_ids }, + "job_ids": job_ids, + } + })) + .execute(db) + .await?; + for (_, child) in steps { + insert_job(db, *child, Some(parent)).await?; + } + Ok(parent) +} + +async fn parent_state(db: &Pool, parent: Uuid) -> anyhow::Result<(i32, bool, Value)> { + let (suspend, parked, status): (i32, bool, Value) = sqlx::query_as( + "SELECT q.suspend, q.suspend_until IS NOT NULL, s.workflow_as_code_status \ + FROM v2_job_queue q JOIN v2_job_status s USING (id) WHERE q.id = $1", + ) + .bind(parent) + .fetch_one(db) + .await?; + Ok((suspend, parked, status)) +} + +/// The zombie monitor's path: `handle_job_error` → `add_completed_job_error`, never +/// the worker's result processor. The parent must come out of it pullable, with the +/// failure recorded under the step so the workflow's `try/catch` sees a task error. +#[sqlx::test(fixtures("base"))] +async fn a_child_failed_outside_the_worker_wakes_its_parent( + db: Pool, +) -> anyhow::Result<()> { + let child = Uuid::new_v4(); + let parent = plant_parked_parent(&db, &[("slowTask", child)]).await?; + let child_job = get_mini_completed_job(&child, W_ID, &db).await?.unwrap(); + + add_completed_job_error( + &db, + &child_job, + 0, + None, + json!({"name": "ExecutionErr", "message": "Job timed out after no ping"}), + "monitor", + false, + None, + ) + .await?; + + let (suspend, parked, status) = parent_state(&db, parent).await?; + assert_eq!(suspend, 0, "the parent must be released"); + assert!( + parked, + "suspend_until stays set: the suspended pull query keys on it" + ); + let step = &status["_checkpoint"]["completed_steps"]["slowTask"]; + assert_eq!(step["__wmill_error"], json!(true), "{status}"); + assert_eq!(step["child_job_id"], json!(child.to_string())); + assert_eq!( + step["result"]["error"]["message"], + json!("Job timed out after no ping") + ); + assert!( + status["_checkpoint"].get("pending_steps").is_none(), + "nothing left to wait on: {status}" + ); + assert!( + windmill_common::wac::WAC_SUSPEND_READY.swap(false, std::sync::atomic::Ordering::Relaxed) + ); + Ok(()) +} + +/// Only a child the parent is waiting on moves the counter. A child the body +/// launched itself, or a completion arriving after the key was re-dispatched to +/// another job, records its timeline entry and nothing else. +#[sqlx::test(fixtures("base"))] +async fn a_child_the_parent_is_not_waiting_on_leaves_it_parked( + db: Pool, +) -> anyhow::Result<()> { + let awaited = Uuid::new_v4(); + let parent = plant_parked_parent(&db, &[("task", awaited)]).await?; + let stray = Uuid::new_v4(); + insert_job(&db, stray, Some(parent)).await?; + + let stray_job = get_mini_completed_job(&stray, W_ID, &db).await?.unwrap(); + add_completed_job_error( + &db, + &stray_job, + 0, + None, + json!({"message": "boom"}), + "w", + false, + None, + ) + .await?; + + let (suspend, _, status) = parent_state(&db, parent).await?; + assert_eq!( + suspend, 1, + "a stray child must not release the parent: {status}" + ); + assert_eq!(status["_checkpoint"]["completed_steps"], json!({})); + assert!( + status[stray.to_string()]["duration_ms"].is_number(), + "the timeline entry is still stamped: {status}" + ); + + let awaited_job = get_mini_completed_job(&awaited, W_ID, &db).await?.unwrap(); + let result = serde_json::value::to_raw_value(&json!("done"))?; + add_completed_job( + &db, + &awaited_job, + true, + false, + Json(&result), + None, + 0, + None, + false, + None, + false, + ) + .await?; + + let (suspend, _, status) = parent_state(&db, parent).await?; + assert_eq!(suspend, 0); + assert_eq!( + status["_checkpoint"]["completed_steps"]["task"], + json!("done") + ); + Ok(()) +} diff --git a/backend/windmill-common/src/wac.rs b/backend/windmill-common/src/wac.rs index d46c937e33..05c343b427 100644 --- a/backend/windmill-common/src/wac.rs +++ b/backend/windmill-common/src/wac.rs @@ -8,11 +8,17 @@ use serde::{Deserialize, Serialize}; use serde_json::Value; use sqlx::{Postgres, Transaction}; +use std::sync::atomic::AtomicBool; use uuid::Uuid; use crate::error::{self, Error}; use crate::DB; +/// Set when a child completion brought a parked WAC parent's `suspend` counter to +/// zero, so a worker's pull loop tries the suspended-jobs query first instead of +/// waiting for its next periodic attempt. +pub static WAC_SUSPEND_READY: AtomicBool = AtomicBool::new(false); + /// Checkpoint state persisted across workflow invocations. #[derive(Debug, Serialize, Deserialize, Default, Clone)] pub struct WacCheckpoint { @@ -672,3 +678,199 @@ pub async fn persist_inline_checkpoint_delta( Ok(failure) } + +/// The step key a WAC v2 parent is waiting on `child_job` for, if any. +/// +/// `job_ids` is the parent's `pending_steps.job_ids` (step key → child job id). +/// A child absent from it is not a step this round is waiting on: a completion +/// arriving after the parent re-dispatched the key to a new job, or a child the +/// body launched directly (`runScript` and friends). +fn pending_step_key(job_ids: &Value, child_job: &Uuid) -> Option { + let child = child_job.to_string(); + job_ids + .as_object()? + .iter() + .find_map(|(key, id)| (id.as_str() == Some(child.as_str())).then(|| key.clone())) +} + +/// Record a completed child on its WAC parent, in the child's completion transaction. +/// +/// Every path that brings a child to a terminal state — a worker's result, the +/// zombie monitor, a cancel — completes it through `add_completed_job`, and this is +/// where the parent learns of it. For a child the parent is parked on, the step's +/// result (or failure record) is merged into the checkpoint's `completed_steps` and +/// the parent's `suspend` counter drops by one, atomically with the child's own +/// completion: there is no window in which the child is completed but the parent +/// still waits for it. For any other child, only the timeline entry is stamped. +/// +/// Returns whether the counter reached zero, i.e. the parent is ready to be pulled. +/// +/// Exactly once: the merge is refused when `completed_steps` already holds the key +/// or `job_ids` no longer maps it to this child, and the decrement follows only a +/// merge that happened. Two completions of one child (a worker and the monitor +/// racing) therefore decrement once, and a stale completion never touches a +/// counter that belongs to a later round. +/// +/// Lock order: the parent's queue row, then its status row, then (by the caller) +/// the child's queue row. A cancel walks parent then children, the parent's own +/// completion deletes its queue row and cascades to its status row, and the park +/// (`suspend_wac_parent`) locks the queue row before writing the checkpoint, so +/// any other order can deadlock against one of them. +/// +/// Authorization: none is checked here. `child_job` is the job the caller is +/// completing, which it already holds, and `parent_job` must be that job's +/// persisted `v2_job.parent_job` (both callers read it from the child's row). +/// The read that gates the step merge and the counter decrement joins on that +/// relationship, so a mismatched pair changes no parent's `completed_steps` or +/// `suspend`; only the timeline stamp at the end runs unconditionally. Job ids +/// are global, so no workspace scoping is needed on top. +pub async fn record_child_completion( + tx: &mut Transaction<'_, Postgres>, + parent_job: &Uuid, + child_job: &Uuid, + success: bool, + duration_ms: i64, + result: &str, +) -> error::Result { + let job_ids: Option> = sqlx::query_scalar( + "SELECT s.workflow_as_code_status->'_checkpoint'->'pending_steps'->'job_ids' \ + FROM v2_job_status s JOIN v2_job c ON c.parent_job = s.id \ + WHERE s.id = $1 AND c.id = $2", + ) + .bind(parent_job) + .bind(child_job) + .fetch_optional(&mut **tx) + .await + .map_err(|e| Error::internal_err(format!("Failed to read WAC parent {parent_job}: {e}")))?; + + let step_key = job_ids + .flatten() + .and_then(|ids| pending_step_key(&ids, child_job)); + + let mut parent_ready = false; + if let Some(step_key) = step_key { + let parked: Option = + sqlx::query_scalar("SELECT suspend FROM v2_job_queue WHERE id = $1 FOR UPDATE") + .bind(parent_job) + .fetch_optional(&mut **tx) + .await + .map_err(|e| { + Error::internal_err(format!("Failed to lock WAC parent {parent_job}: {e}")) + })?; + + if parked.is_some() { + let step_value = if success { + result.to_string() + } else { + let raw: Value = serde_json::from_str(result).unwrap_or(Value::Null); + wac_failure_record(&step_key, Some(&child_job.to_string()), &raw).to_string() + }; + let merged: Option = sqlx::query_scalar( + "UPDATE v2_job_status SET workflow_as_code_status = jsonb_set( + workflow_as_code_status, + '{_checkpoint,completed_steps}', + COALESCE(workflow_as_code_status->'_checkpoint'->'completed_steps', '{}'::jsonb) + || jsonb_build_object($2::text, $3::text::jsonb) + ) WHERE id = $1 + AND workflow_as_code_status->'_checkpoint'->'pending_steps'->'job_ids'->>$2 = $4 + AND NOT COALESCE(workflow_as_code_status->'_checkpoint'->'completed_steps' ? $2, false) + RETURNING 1", + ) + .bind(parent_job) + .bind(&step_key) + .bind(&step_value) + .bind(child_job.to_string()) + .fetch_optional(&mut **tx) + .await + .map_err(|e| { + Error::internal_err(format!("Failed to add WAC completed step: {e}")) + })?; + + if merged.is_some() { + // `suspend_until` stays set: the suspended pull query is what takes a + // parked parent back, and it selects on `suspend_until IS NOT NULL`. + let suspend: Option = sqlx::query_scalar( + "UPDATE v2_job_queue SET suspend = GREATEST(suspend - 1, 0) \ + WHERE id = $1 RETURNING suspend", + ) + .bind(parent_job) + .fetch_optional(&mut **tx) + .await + .map_err(|e| Error::internal_err(format!("Failed to unsuspend WAC parent: {e}")))?; + parent_ready = suspend == Some(0); + if parent_ready { + sqlx::query( + "UPDATE v2_job_status SET workflow_as_code_status = \ + workflow_as_code_status #- '{_checkpoint,pending_steps}' WHERE id = $1", + ) + .bind(parent_job) + .execute(&mut **tx) + .await + .map_err(|e| { + Error::internal_err(format!("Failed to clear WAC pending steps: {e}")) + })?; + } + } + tracing::info!( + parent_job = %parent_job, + child_job = %child_job, + step_key = %step_key, + success, + recorded = merged.is_some(), + parent_ready, + "WAC v2 child job completed" + ); + } + } + + // The child's entry in the parent's timeline, keyed by child id. The parent may + // already be completed (cancelled with its children still running), in which + // case the entry lives on its completed row instead. Errors propagate: a failed + // statement has already aborted the transaction, so there is nothing to continue with. + let stamped: Option = sqlx::query_scalar( + "UPDATE v2_job_status SET workflow_as_code_status = jsonb_set( + jsonb_set( + workflow_as_code_status, + ARRAY[$1], + COALESCE(workflow_as_code_status->$1, '{}'::jsonb) + ), + ARRAY[$1, 'duration_ms'], + to_jsonb($2::bigint) + ) WHERE id = $3 AND workflow_as_code_status IS NOT NULL RETURNING 1", + ) + .bind(child_job.to_string()) + .bind(duration_ms) + .bind(parent_job) + .fetch_optional(&mut **tx) + .await + .map_err(|e| { + Error::internal_err(format!( + "Could not update parent job `duration_ms` in workflow as code status: {e}" + )) + })?; + if stamped.is_none() { + sqlx::query( + "UPDATE v2_job_completed SET workflow_as_code_status = jsonb_set( + jsonb_set( + workflow_as_code_status, + ARRAY[$1], + COALESCE(workflow_as_code_status->$1, '{}'::jsonb) + ), + ARRAY[$1, 'duration_ms'], + to_jsonb($2::bigint) + ) WHERE id = $3 AND workflow_as_code_status IS NOT NULL", + ) + .bind(child_job.to_string()) + .bind(duration_ms) + .bind(parent_job) + .execute(&mut **tx) + .await + .map_err(|e| { + Error::internal_err(format!( + "Could not update completed parent job `duration_ms` in workflow as code status: {e}" + )) + })?; + } + + Ok(parent_ready) +} diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index b6380c02d0..c4058dba6f 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -983,7 +983,7 @@ pub async fn add_completed_job( flow_is_done: bool, duration: Option, from_cache: bool, -) -> Result<(Uuid, i64, Option), Error> { +) -> Result<(Uuid, i64), Error> { // tracing::error!("Start"); // let start = tokio::time::Instant::now(); @@ -1017,7 +1017,7 @@ pub async fn add_completed_job( }; let result_columns = result_columns.as_ref(); - let (opt_uuid, duration, _skip_downstream_error_handlers, wac_job_ids) = (|| { + let (opt_uuid, duration, _skip_downstream_error_handlers, wac_parent_ready) = (|| { commit_completed_job( db, completed_job, @@ -1052,9 +1052,13 @@ pub async fn add_completed_job( .sleep(tokio::time::sleep) .await?; + if wac_parent_ready { + windmill_common::wac::WAC_SUSPEND_READY.store(true, std::sync::atomic::Ordering::Relaxed); + } + // if scheduling next job failed, return the job_id early to ensure the job get retried after a timeout if let Some(job_id) = opt_uuid { - return Ok((job_id, duration, None)); + return Ok((job_id, duration)); } // Auto-resolve a retry chain that ultimately worked, from whichever of the two @@ -1101,7 +1105,7 @@ pub async fn add_completed_job( // tracing::error!("4 {:?}", start.elapsed()); - Ok((completed_job.id, duration, wac_job_ids)) + Ok((completed_job.id, duration)) } async fn commit_completed_job( @@ -1119,7 +1123,7 @@ async fn commit_completed_job( // True when a native script retry was enqueued for this failed attempt, i.e. // this is not the terminal attempt — schedule completion handlers must wait. retry_pending: bool, -) -> windmill_common::error::Result<(Option, i64, bool, Option)> { +) -> windmill_common::error::Result<(Option, i64, bool, bool)> { // let start = std::time::Instant::now(); let job_id = completed_job.id; @@ -1249,74 +1253,23 @@ async fn commit_completed_job( .map_err(|e| Error::InternalErr(format!("Could not update job labels: {e:#}")))?; } - let mut wac_job_ids: Option = None; + // Before `delete_job`: the parent's rows are locked ahead of the child's own + // queue row (see `record_child_completion` for the order this must keep). + let mut wac_parent_ready = false; if !completed_job.is_flow_step() { if let Some(parent_job) = completed_job.parent_job { - // Only update WAC parents (v1 or v2). The WHERE condition skips - // non-WAC parents entirely (error handlers, run_script children, etc.). - // Also returns pending_steps.job_ids so WAC v2 child completion - // doesn't need a separate read. - let row = sqlx::query_scalar!( - r#"UPDATE v2_job_status SET - workflow_as_code_status = jsonb_set( - jsonb_set( - workflow_as_code_status, - array[$1], - COALESCE(workflow_as_code_status->$1, '{}'::jsonb) - ), - array[$1, 'duration_ms'], - to_jsonb($2::bigint) - ) - WHERE id = $3 AND workflow_as_code_status IS NOT NULL - RETURNING workflow_as_code_status->'_checkpoint'->'pending_steps'->'job_ids' AS "job_ids: serde_json::Value""#, - &completed_job.id.to_string(), + wac_parent_ready = windmill_common::wac::record_child_completion( + &mut tx, + &parent_job, + &completed_job.id, + success, duration, - parent_job + sanitized_result.as_ref(), ) - .fetch_optional(&mut *tx) .warn_after_seconds(10) - .await - .inspect_err(|e| { - tracing::error!( - "Could not update parent job `duration_ms` in workflow as code status: {}", - e, - ) - }) - .ok() - .flatten(); - wac_job_ids = row.flatten(); - - // If parent was already completed (e.g. cancelled), update v2_job_completed instead - if wac_job_ids.is_none() { - let _ = sqlx::query!( - r#"UPDATE v2_job_completed SET - workflow_as_code_status = jsonb_set( - jsonb_set( - workflow_as_code_status, - array[$1], - COALESCE(workflow_as_code_status->$1, '{}'::jsonb) - ), - array[$1, 'duration_ms'], - to_jsonb($2::bigint) - ) - WHERE id = $3 AND workflow_as_code_status IS NOT NULL"#, - &completed_job.id.to_string(), - duration, - parent_job - ) - .execute(&mut *tx) - .warn_after_seconds(10) - .await - .inspect_err(|e| { - tracing::error!( - "Could not update completed parent job `duration_ms` in workflow as code status: {}", - e, - ) - }); - } + .await?; } } - // tracing::error!("Added completed job {:#?}", queued_job); let mut _skip_downstream_error_handlers = false; tx = delete_job(tx, &job_id).warn_after_seconds(10).await?; @@ -1544,14 +1497,19 @@ async fn commit_completed_job( completed_job.id ); // tracing::info!("completed job: {:?}", start.elapsed().as_micros()); - Ok((None, duration, _skip_downstream_error_handlers, wac_job_ids)) + Ok(( + None, + duration, + _skip_downstream_error_handlers, + wac_parent_ready, + )) } async fn check_result_size( db: &Pool, queued_job: &MiniCompletedJob, result: Json<&T>, -) -> Option, i64, bool, Option), Error>> { +) -> Option, i64, bool, bool), Error>> { let result_size = result.size() / 1024 / 1024; if result_size > 2 { if result_size > *MAX_RESULT_SIZE_MB { diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index a034b0d184..4c19d0510d 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -2918,6 +2918,26 @@ pub async fn handle_wac_v2_output( { let mut tx = db.begin().await?; + // Park before writing the checkpoint. This locks the queue row ahead of the + // status row, the order `record_child_completion` takes, so a stale child + // finishing while the parent re-dispatches cannot deadlock this transaction. + // A cancel already on the row is also seen before anything is written, and + // every child pushed after the commit finds a parked parent to decrement. + match crate::wac_executor::suspend_wac_parent( + &mut tx, + &job.id, + &job.workspace_id, + num_steps as i32, + 14.0 * 24.0 * 3600.0, + ) + .await? + { + WacPark::Parked(ms) => segment_ms = ms, + WacPark::Cancelled(cancel) => { + return Err(wac_cancelled_mid_segment(cancel, canceled_by)) + } + } + // Update checkpoint with pending steps update_checkpoint_for_dispatch(&mut checkpoint, &steps, &mode, &job_ids); let status_json = serde_json::to_value(&checkpoint).map_err(|e| { @@ -2967,25 +2987,6 @@ pub async fn handle_wac_v2_output( })?; } - // Suspend parent before children become visible, so a child that - // completes immediately finds a parked parent to decrement. - match crate::wac_executor::suspend_wac_parent( - &mut tx, - &job.id, - &job.workspace_id, - num_steps as i32, - 14.0 * 24.0 * 3600.0, - ) - .await? - { - WacPark::Parked(ms) => segment_ms = ms, - // Returning here drops `tx`, unwriting the checkpoint and the timeline - // entries, so no child is ever pushed against a parent that never parked. - WacPark::Cancelled(cancel) => { - return Err(wac_cancelled_mid_segment(cancel, canceled_by)) - } - } - tx.commit().await?; } @@ -3314,6 +3315,23 @@ pub async fn handle_wac_v2_output( let mut tx = db.begin().await?; + // Park first: the queue row is locked before the status row, the order every + // child completion takes. + let segment_ms = match crate::wac_executor::suspend_wac_parent( + &mut tx, + &job.id, + &job.workspace_id, + 1, + timeout_secs, + ) + .await? + { + WacPark::Parked(ms) => ms, + WacPark::Cancelled(cancel) => { + return Err(wac_cancelled_mid_segment(cancel, canceled_by)) + } + }; + // Save checkpoint let status_json = serde_json::to_value(&checkpoint).map_err(|e| { error::Error::internal_err(format!("Failed to serialize checkpoint: {e}")) @@ -3468,22 +3486,6 @@ pub async fn handle_wac_v2_output( })?; } - // Suspend parent with suspend=1 (waiting for 1 approval event) - let segment_ms = match crate::wac_executor::suspend_wac_parent( - &mut tx, - &job.id, - &job.workspace_id, - 1, - timeout_secs, - ) - .await? - { - WacPark::Parked(ms) => ms, - WacPark::Cancelled(cancel) => { - return Err(wac_cancelled_mid_segment(cancel, canceled_by)) - } - }; - tx.commit().await?; crate::wac_executor::end_wac_segment(conn, job, segment_ms); @@ -3521,6 +3523,24 @@ pub async fn handle_wac_v2_output( let mut tx = db.begin().await?; + // Park first: the queue row is locked before the status row, the order every + // child completion takes. suspend=1 (not 0) so the suspended pull query only + // picks it up when `suspend_until <= now()`, not via `suspend <= 0`. + let segment_ms = match crate::wac_executor::suspend_wac_parent( + &mut tx, + &job.id, + &job.workspace_id, + 1, + sleep_secs, + ) + .await? + { + WacPark::Parked(ms) => ms, + WacPark::Cancelled(cancel) => { + return Err(wac_cancelled_mid_segment(cancel, canceled_by)) + } + }; + // Save checkpoint let status_json = serde_json::to_value(&checkpoint).map_err(|e| { error::Error::internal_err(format!("Failed to serialize checkpoint: {e}")) @@ -3569,23 +3589,6 @@ pub async fn handle_wac_v2_output( })?; } - // Use suspend=1 (not 0) so the suspended pull query only picks it up - // when `suspend_until <= now()`, not via `suspend <= 0`. - let segment_ms = match crate::wac_executor::suspend_wac_parent( - &mut tx, - &job.id, - &job.workspace_id, - 1, - sleep_secs, - ) - .await? - { - WacPark::Parked(ms) => ms, - WacPark::Cancelled(cancel) => { - return Err(wac_cancelled_mid_segment(cancel, canceled_by)) - } - }; - tx.commit().await?; crate::wac_executor::end_wac_segment(conn, job, segment_ms); @@ -3623,21 +3626,12 @@ pub async fn handle_wac_v2_output( let source_hash = job.runnable_id.map(|h| h.0.to_string()); let mut tx = db.begin().await?; - crate::wac_executor::persist_inline_checkpoint_delta( - &mut tx, - &job.id, - source_hash.as_deref(), - &key, - value, - started_at.as_deref(), - duration_ms, - ) - .await?; - // Reset running=false so the job is immediately eligible for pickup. // Unlike dispatch (which sets suspend>0), inline checkpoints don't suspend — // the job should be re-run right away to continue past the cached step. // `prev` holds the pre-update row: RETURNING would see the cleared column. + // Runs before the checkpoint write so the queue row is locked ahead of the + // status row, the order every child completion takes. let segment_ms = sqlx::query_scalar!( "WITH prev AS (SELECT started_at FROM v2_job_queue WHERE id = $1) UPDATE v2_job_queue q SET running = false, started_at = null @@ -3654,6 +3648,17 @@ pub async fn handle_wac_v2_output( })? .flatten(); + crate::wac_executor::persist_inline_checkpoint_delta( + &mut tx, + &job.id, + source_hash.as_deref(), + &key, + value, + started_at.as_deref(), + duration_ms, + ) + .await?; + tx.commit().await?; crate::wac_executor::end_wac_segment(conn, job, segment_ms); diff --git a/backend/windmill-worker/src/result_processor.rs b/backend/windmill-worker/src/result_processor.rs index 21b83b32ff..a269e72f46 100644 --- a/backend/windmill-worker/src/result_processor.rs +++ b/backend/windmill-worker/src/result_processor.rs @@ -16,10 +16,6 @@ use windmill_common::otel_oss::FutureExt; use uuid::Uuid; -/// Set by the result processor when a WAC child completion makes suspend reach 0, -/// signaling the worker main loop to check for suspended jobs immediately. -pub static WAC_SUSPEND_READY: AtomicBool = AtomicBool::new(false); - use windmill_common::{ add_time, error::{self, Error}, @@ -1794,7 +1790,7 @@ pub async fn process_completed_job( add_time!(bench, "pre add_completed_job"); - let (_, duration, wac_job_ids) = add_completed_job( + let (_, duration) = add_completed_job( db, &job, true, @@ -1875,29 +1871,6 @@ pub async fn process_completed_job( } return Ok(r); } - } else if let Some(parent_job) = parent_job { - // wac_job_ids is piggybacked from the duration write in - // add_completed_job — no extra query needed. - if let Some(job_ids) = wac_job_ids { - if let Ok(Some(_)) = handle_wac_child_completion( - db, - &job_id, - parent_job, - &workspace_id, - result, - true, - job_ids, - ) - .await - { - if let Some(done_tx) = done_tx { - done_tx - .send(()) - .expect("done receiver should still be alive"); - } - return Ok(None); - } - } } } else { // The result already carries our injected @@ -1994,227 +1967,11 @@ pub async fn process_completed_job( } return Ok(r); } - } else if let Some(parent_job) = job.parent_job { - // WAC child failed — query job_ids from parent (errors are rare, - // so the extra read is acceptable here). - let job_ids_json: Option> = sqlx::query_scalar( - "SELECT workflow_as_code_status->'_checkpoint'->'pending_steps'->'job_ids' \ - FROM v2_job_status WHERE id = $1", - ) - .bind(&parent_job) - .fetch_optional(db) - .await?; - if let Some(Some(job_ids)) = job_ids_json { - if let Ok(Some(_)) = handle_wac_child_completion( - db, - &job.id, - parent_job, - &job.workspace_id, - downstream_result, - false, - job_ids, - ) - .await - { - if let Some(done_tx) = done_tx { - done_tx - .send(()) - .expect("done receiver should still be alive"); - } - return Ok(None); - } - } } } return Ok(None); } -/// Handle a WAC v2 child job completion. -/// Returns Ok(Some(())) if the parent was a WAC job and was handled, -/// Ok(None) if the parent is not a WAC job (caller should fall through). -/// -/// CONCURRENCY: Multiple parallel children may complete simultaneously on -/// different workers. We use atomic SQL operations throughout: -/// - `completed_steps` is merged via `jsonb_set(... || jsonb_build_object(...))` -/// — PostgreSQL serialises concurrent UPDATEs on the same row, so each -/// worker sees the previous worker's writes. -/// - The suspend counter (set to N at dispatch time) is decremented atomically -/// with `RETURNING` to determine the "all done" condition. -pub(crate) async fn handle_wac_child_completion( - db: &DB, - child_job_id: &Uuid, - parent_job_id: Uuid, - workspace_id: &str, - result: Arc>, - success: bool, - job_ids_value: Value, -) -> error::Result> { - let job_ids = match job_ids_value { - Value::Object(m) => m, - _ => return Ok(None), // Not a WAC parent or no pending steps - }; - - let child_id_str = child_job_id.to_string(); - let step_key = job_ids.iter().find_map(|(key, val)| { - if val.as_str() == Some(&child_id_str) { - Some(key.clone()) - } else { - None - } - }); - - let step_key = match step_key { - Some(k) => k, - None => { - if !success { - // No step key and failed — can't store error, fail parent immediately - tracing::error!( - parent_job = %parent_job_id, - child_job = %child_job_id, - "WAC v2 child job failed but no step key found, failing parent" - ); - sqlx::query!( - "UPDATE v2_job_queue SET suspend = 0, suspend_until = NULL WHERE id = $1", - parent_job_id, - ) - .execute(db) - .await?; - let parent_mini = get_mini_completed_job(&parent_job_id, workspace_id, db).await?; - if let Some(parent_mini) = parent_mini { - let child_err: Value = - serde_json::from_str(result.get()).unwrap_or(Value::Null); - let err_value = json!({ - "message": format!("WAC child job {} failed (no step key)", child_job_id), - "error": child_err, - }); - let _ = windmill_queue::add_completed_job_error( - db, - &parent_mini, - 0, - None, - err_value, - "wac_child_handler", - false, - None, - ) - .await; - } - return Ok(Some(())); - } - tracing::warn!( - parent_job = %parent_job_id, - child_job = %child_job_id, - "WAC v2 child completed but no matching step key found in checkpoint, decrementing suspend to avoid parent hang" - ); - // Still decrement suspend so the parent doesn't hang indefinitely - let _ = sqlx::query_scalar!( - "UPDATE v2_job_queue \ - SET suspend = GREATEST(suspend - 1, 0) \ - WHERE id = $1 \ - RETURNING suspend", - parent_job_id, - ) - .fetch_optional(db) - .await?; - return Ok(Some(())); - } - }; - - // Build result — wrap errors with _error marker so workflow try/catch can handle them - let result_value: Value = if success { - serde_json::from_str(result.get()).unwrap_or(Value::Null) - } else { - let child_err: Value = serde_json::from_str(result.get()).unwrap_or(Value::Null); - tracing::info!( - parent_job = %parent_job_id, - child_job = %child_job_id, - step_key = %step_key, - "WAC v2 child job failed, storing error for workflow try/catch" - ); - windmill_common::wac::wac_failure_record( - &step_key, - Some(&child_job_id.to_string()), - &child_err, - ) - }; - - tracing::info!( - parent_job = %parent_job_id, - child_job = %child_job_id, - step_key = %step_key, - success = success, - "WAC v2 child job completed" - ); - - // Use a transaction to ensure completed_steps merge + suspend decrement - // are atomic. Without this, a crash between the two could strand the parent. - let result_json = serde_json::to_value(&result_value) - .map_err(|e| error::Error::InternalErr(format!("Failed to serialize step result: {e}")))?; - - let mut tx = db.begin().await?; - - // Merge the completed step into the checkpoint. - // Uses `|| jsonb_build_object(key, value)` so concurrent children on - // different workers don't overwrite each other — PostgreSQL serialises - // concurrent UPDATEs on the same row and each sees the previous write. - sqlx::query( - "UPDATE v2_job_status SET workflow_as_code_status = jsonb_set( - workflow_as_code_status, - '{_checkpoint,completed_steps}', - COALESCE(workflow_as_code_status->'_checkpoint'->'completed_steps', '{}'::jsonb) - || jsonb_build_object($2::text, $3::jsonb) - ) WHERE id = $1", - ) - .bind(&parent_job_id) - .bind(&step_key) - .bind(&result_json) - .execute(&mut *tx) - .await - .map_err(|e| error::Error::InternalErr(format!("Failed to add WAC completed step: {e}")))?; - - // Decrement the suspend counter. The counter was set to N (number of - // children) at dispatch time. When it reaches 0 all children are done. - // Keep suspend_until non-null so the suspended pull query - // (`WHERE suspend_until IS NOT NULL AND suspend <= 0`) picks up the parent. - let new_suspend: Option = sqlx::query_scalar!( - "UPDATE v2_job_queue \ - SET suspend = GREATEST(suspend - 1, 0) \ - WHERE id = $1 \ - RETURNING suspend", - parent_job_id, - ) - .fetch_optional(&mut *tx) - .await?; - - let all_done = new_suspend == Some(0); - - if all_done { - // Clear pending_steps from checkpoint since all children are complete. - // This is cosmetic — the next replay will overwrite it anyway — but - // keeps the checkpoint clean for frontend display. - let _ = sqlx::query( - "UPDATE v2_job_status SET workflow_as_code_status = \ - workflow_as_code_status #- '{_checkpoint,pending_steps}' \ - WHERE id = $1", - ) - .bind(&parent_job_id) - .execute(&mut *tx) - .await; - } - - tx.commit().await?; - - if all_done { - tracing::info!( - parent_job = %parent_job_id, - "WAC v2 all child jobs completed, unsuspending parent" - ); - WAC_SUSPEND_READY.store(true, Ordering::Relaxed); - } - - Ok(Some(())) -} - pub async fn handle_non_flow_job_error( db: &DB, job: &MiniCompletedJob, diff --git a/backend/windmill-worker/src/wac_executor.rs b/backend/windmill-worker/src/wac_executor.rs index 798cd347b1..c188792a5e 100644 --- a/backend/windmill-worker/src/wac_executor.rs +++ b/backend/windmill-worker/src/wac_executor.rs @@ -112,6 +112,11 @@ pub enum WacPark { /// completes a job without a worker-measured duration — a cancel, the child-failure /// handler — falls back to `now() - started_at`. Left pointing at the first segment, /// that fallback reports the whole sleep or approval wait as execution time. +/// +/// Call it before any write to the parent's `v2_job_status` row in the same +/// transaction: a child's completion locks the queue row and then the status row +/// (`record_child_completion`), and taking them the other way round here can +/// deadlock against a stale child finishing while the parent re-dispatches. pub async fn suspend_wac_parent( tx: &mut Transaction<'_, Postgres>, job_id: &Uuid, diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index d893ec071b..467b8dbf19 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -3405,7 +3405,7 @@ pub async fn run_worker( let suspend_first = suspend_first_success || rand::random::() < likelihood_of_suspend || last_suspend_first.elapsed().as_secs_f64() > 5.0 - || crate::result_processor::WAC_SUSPEND_READY + || windmill_common::wac::WAC_SUSPEND_READY .swap(false, Ordering::Relaxed); if suspend_first { diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index 247e618dab..cbda50c7bc 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -2034,8 +2034,8 @@ pub async fn update_flow_status_after_job_completion_internal( chat_ai_info.conversation_id, ) .await?; - let (duration, wac_job_ids) = if success { - let (_, duration, wac_job_ids) = add_completed_job( + let duration = if success { + let (_, duration) = add_completed_job( db, &cflow_job, true, @@ -2049,9 +2049,9 @@ pub async fn update_flow_status_after_job_completion_internal( false, ) .await?; - (duration, wac_job_ids) + duration } else { - let (_, duration, wac_job_ids) = add_completed_job( + let (_, duration) = add_completed_job( db, &cflow_job, false, @@ -2069,30 +2069,11 @@ pub async fn update_flow_status_after_job_completion_internal( false, ) .await?; - (duration, wac_job_ids) + duration }; flow_job_duration = flow_job .started_at .map(|x| FlowJobDuration { started_at: x, duration_ms: duration }); - - // If this flow is a WAC child (not a flow step, has parent), - // notify the WAC parent of completion. - if !flow_job.is_flow_step() { - if let Some(parent_job) = flow_job.parent_job { - if let Some(job_ids) = wac_job_ids { - let _ = crate::result_processor::handle_wac_child_completion( - db, - &flow_job.id, - parent_job, - &flow_job.workspace_id, - nresult.clone(), - success, - job_ids, - ) - .await; - } - } - } } true } else { From 91e6dc39ce795fafc2bed0d799b62c9880fd6430 Mon Sep 17 00:00:00 2001 From: Alexander Petric Date: Mon, 14 Sep 2026 16:04:58 -0400 Subject: [PATCH 09/44] feat: pre-approved cloud accounts: login links, OAuth adoption, setup, and the trial bridge (#10875) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: single-use login links and oauth-claimable pending accounts * docs: capture the auth surface facts behind login links * fix: accept stringified email_verified from oauth userinfo * docs: describe the oauth claim rule in the auth surface notes * fix: harden login-link redirects and sweep expired links * chore: bump ee-repo-ref * fix: keep expired login links a day so an open still reads as expired * fix: refuse login links for superadmin and devops accounts * fix: re-check the account's roles when a login link is opened * feat: pre-approved cloud accounts finish their setup and start their trial from Windmill * feat: dev-only localStorage opt-in to the cloud UI on localhost * feat: finish-setup entry in the desktop settings menu * style: pulse the settings row while account setup is pending; shorter, blue finish-setup entry * fix: list the configured providers in the finish-setup modal * fix: open the finish-setup modal after the menu has closed * feat: finish-setup provider sign-in keeps the session when the provider asserts another address * chore: pin the EE companion commit * fix: plain toast for the finish-setup refusal * style: format the dev cloud override Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_011HMniEf5hapoKEB6TEBcGy * feat: onboarding skips the source question an invite already answered Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_011HMniEf5hapoKEB6TEBcGy * fix: type the finish-setup icons and login_type as the frontend uses them Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011HMniEf5hapoKEB6TEBcGy * feat: invited accounts get a workspace name, hub picks and starter prompts from their invite Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011HMniEf5hapoKEB6TEBcGy * fix: the workspace form reads the invite's name itself, so the picker prefills it too Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011HMniEf5hapoKEB6TEBcGy * feat: an empty workspace offers the projects its invite picked, one click from importing Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011HMniEf5hapoKEB6TEBcGy * style: picked projects get identical import buttons Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011HMniEf5hapoKEB6TEBcGy * feat: a pinned sidebar banner until an invited account has credentials of its own Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011HMniEf5hapoKEB6TEBcGy * style: the account-setup row speaks the rail's language, tinted not filled Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011HMniEf5hapoKEB6TEBcGy * chore: pin ee-repo-ref to the import fix Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011HMniEf5hapoKEB6TEBcGy * refactor: picked projects live in the template picker only; account-setup row moves to the rail footer Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011HMniEf5hapoKEB6TEBcGy * fix: review round — no portal login for job tokens, finish-setup failures keep the session, prompt labels deduped Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011HMniEf5hapoKEB6TEBcGy * fix: CI round — trial start is a POST, profile cache follows the session, setup row on MenuButton Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011HMniEf5hapoKEB6TEBcGy * fix: CI round — no password road where password login is off, cache note on the login form, trial refusal surfaced Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011HMniEf5hapoKEB6TEBcGy * fix: CI round — set_password guarded on its read, refusal stays on the page, docs and formatting Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011HMniEf5hapoKEB6TEBcGy * fix: CI round — popup OAuth clears the profile cache, portal helper crate-private, refusal toast stays Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011HMniEf5hapoKEB6TEBcGy * fix: a refused trial is recorded inline in the rail, not in a day-long toast Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011HMniEf5hapoKEB6TEBcGy * fix: the refusal notice uses the rail's button and has a collapsed form Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011HMniEf5hapoKEB6TEBcGy * fix: CI round — SSO can finish account setup, with the same mismatch refusal as OAuth Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011HMniEf5hapoKEB6TEBcGy * fix: SSO finish-setup rides in RelayState and the refusal notice is a status region Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011HMniEf5hapoKEB6TEBcGy * fix: keep the finish-setup cookie beside RelayState, hoist the status region, pin session-keyed profile cache Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011HMniEf5hapoKEB6TEBcGy * fix: empty live region for the trial refusal, drop the setup cookie once adopted, telemetry inventory Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011HMniEf5hapoKEB6TEBcGy * fix: the trial refusal survives the responsive sidebar swap Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011HMniEf5hapoKEB6TEBcGy * fix: the trial refusal is shown to the account it answers, modal open prop is required Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011HMniEf5hapoKEB6TEBcGy * feat: an invited account skips the whole onboarding survey Its source is the invite and its use case was researched before the invite went out, so neither question is asked: the known source is recorded and onboarding opens on naming the workspace. Accounts without an invite profile see the survey exactly as before. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011HMniEf5hapoKEB6TEBcGy * fix: an invited account with a workspace leaves onboarding before anything paints Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011HMniEf5hapoKEB6TEBcGy * fix: account-setup state resets on sign-out, onboarding shows a loading state while it settles Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011HMniEf5hapoKEB6TEBcGy * style: keep the refresh doc comment on refresh Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011HMniEf5hapoKEB6TEBcGy * fix: profile lists are distinct, and the offer table notes what a users-import does to it Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_011HMniEf5hapoKEB6TEBcGy * chore: update ee-repo-ref to 1ba6fe83451f0a1f8fafe04b7187087d51e0f769 This commit updates the EE repository reference after PR #750 was merged in windmill-ee-private. Previous ee-repo-ref: be42722d09832ffff709a1f710f3e97e34d513b2 New ee-repo-ref: 1ba6fe83451f0a1f8fafe04b7187087d51e0f769 Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Fable 5 Co-authored-by: windmill-internal-app[bot] --- AGENTS.md | 3 + ...9bed4ebbf6a03ec5988acf072c83818d57a02.json | 18 + ...c2a38b2dcf3976fbd67522c4644ee9bddc330.json | 14 + ...5f42477fc5ab17fac70864d1b2f7f91ac7f9d.json | 16 + ...55af2a0ba2e827ae8add59d5e5465dc1d5743.json | 22 + ...0c73aba9b34302e2f25cd6928a29d974bbb2c.json | 22 + ...02455082e0a172948be6441e5383552331c3f.json | 28 + ...1e015ec604eb92a40b8649d054cabef1d8060.json | 22 + ...2b6d22720d71e555501011e1d6b3418064ed0.json | 16 + ...f62e12019aa9582f06b9a47f7d04715eee24c.json | 34 ++ ...1a7760b21c7edb651452c80b46b54ec964901.json | 34 ++ ...cd21ed1fbcc989fcc020c8246d5e2a313b72c.json | 22 + ...3b635507f545a514ff0d9142c663b779dd961.json | 15 + backend/ee-repo-ref.txt | 2 +- .../20260827203157_login_link.down.sql | 1 + .../20260827203157_login_link.up.sql | 13 + .../20260828190901_cloud_trial_offer.down.sql | 1 + .../20260828190901_cloud_trial_offer.up.sql | 11 + ...09213500_cloud_onboarding_profile.down.sql | 1 + ...0909213500_cloud_onboarding_profile.up.sql | 9 + backend/src/monitor.rs | 17 + .../tests/login_link.rs | 192 ++++++ backend/windmill-api-users/src/users.rs | 566 +++++++++++++++++- backend/windmill-api-users/src/users_oss.rs | 10 + backend/windmill-api/openapi.yaml | 197 +++++- backend/windmill-common/src/users.rs | 7 + docs/auth-surface.md | 50 ++ docs/feature-telemetry.md | 4 +- frontend/src/lib/cloud.ts | 9 + .../lib/components/InstanceSettings.svelte | 8 +- frontend/src/lib/components/Login.svelte | 7 + .../src/lib/components/home/HomeAIChat.svelte | 15 +- .../components/home/HubTemplatePicker.svelte | 36 +- .../settings/UserInfoSettings.svelte | 13 +- .../sidebar/AccountSetupBanner.svelte | 48 ++ .../sidebar/FinishAccountSetup.svelte | 173 ++++++ .../lib/components/sidebar/MenuButton.svelte | 9 +- .../components/sidebar/SettingsMenu.svelte | 19 +- .../components/sidebar/SidebarUsage.svelte | 121 +++- .../lib/components/sidebar/UserMenu.svelte | 16 +- .../components/sidebar/accountSetup.svelte.ts | 63 ++ .../SimpleCreateWorkspace.svelte | 18 +- frontend/src/lib/hubProject.test.ts | 28 +- frontend/src/lib/hubProject.ts | 36 ++ frontend/src/lib/logout.ts | 4 + frontend/src/lib/onboardingProfile.test.ts | 54 ++ frontend/src/lib/onboardingProfile.ts | 116 ++++ .../src/routes/(root)/(logged)/+layout.svelte | 30 + .../user/(user)/onboarding/+page.svelte | 69 ++- frontend/src/routes/(root)/+layout.svelte | 2 + .../login_callback/[client_name]/+page.svelte | 27 + .../user/login_link_expired/+page.svelte | 23 + 52 files changed, 2244 insertions(+), 47 deletions(-) create mode 100644 backend/.sqlx/query-071de805623be166dddd2655f099bed4ebbf6a03ec5988acf072c83818d57a02.json create mode 100644 backend/.sqlx/query-25f27dba5c0ea81d9412bdf1986c2a38b2dcf3976fbd67522c4644ee9bddc330.json create mode 100644 backend/.sqlx/query-310d91848c7a032846aa8be8c5e5f42477fc5ab17fac70864d1b2f7f91ac7f9d.json create mode 100644 backend/.sqlx/query-42783d94ee41c5b17ec16b480dd55af2a0ba2e827ae8add59d5e5465dc1d5743.json create mode 100644 backend/.sqlx/query-4ed69ae9e2a0d045ec63e327bc40c73aba9b34302e2f25cd6928a29d974bbb2c.json create mode 100644 backend/.sqlx/query-5bd410d777a7a6d48129e9fee8402455082e0a172948be6441e5383552331c3f.json create mode 100644 backend/.sqlx/query-5ca0afc5a7b0437de221c8cc7b31e015ec604eb92a40b8649d054cabef1d8060.json create mode 100644 backend/.sqlx/query-64bc01a5d88680febabd794b6472b6d22720d71e555501011e1d6b3418064ed0.json create mode 100644 backend/.sqlx/query-754598696e57a8c3ee6477d4f55f62e12019aa9582f06b9a47f7d04715eee24c.json create mode 100644 backend/.sqlx/query-a2be5aeb7e663b0fe403726b4a41a7760b21c7edb651452c80b46b54ec964901.json create mode 100644 backend/.sqlx/query-ab16363a5225b022c7262f3caf5cd21ed1fbcc989fcc020c8246d5e2a313b72c.json create mode 100644 backend/.sqlx/query-b2855a7bf20ec5a405d8c059e7b3b635507f545a514ff0d9142c663b779dd961.json create mode 100644 backend/migrations/20260827203157_login_link.down.sql create mode 100644 backend/migrations/20260827203157_login_link.up.sql create mode 100644 backend/migrations/20260828190901_cloud_trial_offer.down.sql create mode 100644 backend/migrations/20260828190901_cloud_trial_offer.up.sql create mode 100644 backend/migrations/20260909213500_cloud_onboarding_profile.down.sql create mode 100644 backend/migrations/20260909213500_cloud_onboarding_profile.up.sql create mode 100644 backend/windmill-api-integration-tests/tests/login_link.rs create mode 100644 docs/auth-surface.md create mode 100644 frontend/src/lib/components/sidebar/AccountSetupBanner.svelte create mode 100644 frontend/src/lib/components/sidebar/FinishAccountSetup.svelte create mode 100644 frontend/src/lib/components/sidebar/accountSetup.svelte.ts create mode 100644 frontend/src/lib/onboardingProfile.test.ts create mode 100644 frontend/src/lib/onboardingProfile.ts create mode 100644 frontend/src/routes/user/login_link_expired/+page.svelte diff --git a/AGENTS.md b/AGENTS.md index f8c83ec465..8a63c9828a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -30,6 +30,9 @@ Open-source platform for internal tools, workflows, API integrations, background reaches the DB only through the API, so `Connection::Http` paths are never taken by a plain `cargo run`; a normal build cannot start one at all. - **Enterprise**: `docs/enterprise.md` — EE file conventions and PR workflow +- **Auth surface**: `docs/auth-surface.md` — credential precedence, session/cache invalidation + scope, how OAuth login matches `login_type`, and that every superadmin route refuses `$WM_TOKEN`. + Read before designing anything that creates users, tokens or sessions. - **Product telemetry**: `docs/feature-telemetry.md` — when to instrument a new feature with `feature_usage`, and the four-step recipe. An unregistered `(feature, kind)` pair is dropped silently, so frontend-only instrumentation records nothing. diff --git a/backend/.sqlx/query-071de805623be166dddd2655f099bed4ebbf6a03ec5988acf072c83818d57a02.json b/backend/.sqlx/query-071de805623be166dddd2655f099bed4ebbf6a03ec5988acf072c83818d57a02.json new file mode 100644 index 0000000000..0aad412ebe --- /dev/null +++ b/backend/.sqlx/query-071de805623be166dddd2655f099bed4ebbf6a03ec5988acf072c83818d57a02.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO login_link (token_hash, email, rd, expiration, created_by)\n VALUES ($1, $2, $3, $4, $5)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Bpchar", + "Varchar", + "Text", + "Timestamptz", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "071de805623be166dddd2655f099bed4ebbf6a03ec5988acf072c83818d57a02" +} diff --git a/backend/.sqlx/query-25f27dba5c0ea81d9412bdf1986c2a38b2dcf3976fbd67522c4644ee9bddc330.json b/backend/.sqlx/query-25f27dba5c0ea81d9412bdf1986c2a38b2dcf3976fbd67522c4644ee9bddc330.json new file mode 100644 index 0000000000..29ec70e75e --- /dev/null +++ b/backend/.sqlx/query-25f27dba5c0ea81d9412bdf1986c2a38b2dcf3976fbd67522c4644ee9bddc330.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE cloud_trial_offer SET consumed_at = now() WHERE email = $1 AND consumed_at IS NULL", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "25f27dba5c0ea81d9412bdf1986c2a38b2dcf3976fbd67522c4644ee9bddc330" +} diff --git a/backend/.sqlx/query-310d91848c7a032846aa8be8c5e5f42477fc5ab17fac70864d1b2f7f91ac7f9d.json b/backend/.sqlx/query-310d91848c7a032846aa8be8c5e5f42477fc5ab17fac70864d1b2f7f91ac7f9d.json new file mode 100644 index 0000000000..54da0105e7 --- /dev/null +++ b/backend/.sqlx/query-310d91848c7a032846aa8be8c5e5f42477fc5ab17fac70864d1b2f7f91ac7f9d.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE password SET password_hash = $1, login_type = 'password'\n WHERE email = $2 AND login_type = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "310d91848c7a032846aa8be8c5e5f42477fc5ab17fac70864d1b2f7f91ac7f9d" +} diff --git a/backend/.sqlx/query-42783d94ee41c5b17ec16b480dd55af2a0ba2e827ae8add59d5e5465dc1d5743.json b/backend/.sqlx/query-42783d94ee41c5b17ec16b480dd55af2a0ba2e827ae8add59d5e5465dc1d5743.json new file mode 100644 index 0000000000..5522f3bb17 --- /dev/null +++ b/backend/.sqlx/query-42783d94ee41c5b17ec16b480dd55af2a0ba2e827ae8add59d5e5465dc1d5743.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT profile FROM cloud_onboarding_profile WHERE email = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "profile", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "42783d94ee41c5b17ec16b480dd55af2a0ba2e827ae8add59d5e5465dc1d5743" +} diff --git a/backend/.sqlx/query-4ed69ae9e2a0d045ec63e327bc40c73aba9b34302e2f25cd6928a29d974bbb2c.json b/backend/.sqlx/query-4ed69ae9e2a0d045ec63e327bc40c73aba9b34302e2f25cd6928a29d974bbb2c.json new file mode 100644 index 0000000000..f7331a55e8 --- /dev/null +++ b/backend/.sqlx/query-4ed69ae9e2a0d045ec63e327bc40c73aba9b34302e2f25cd6928a29d974bbb2c.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO password(email, verified, password_hash, login_type, super_admin, name, company, username, first_time_user)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Bool", + "Varchar", + "Varchar", + "Bool", + "Varchar", + "Varchar", + "Varchar", + "Bool" + ] + }, + "nullable": [] + }, + "hash": "4ed69ae9e2a0d045ec63e327bc40c73aba9b34302e2f25cd6928a29d974bbb2c" +} diff --git a/backend/.sqlx/query-5bd410d777a7a6d48129e9fee8402455082e0a172948be6441e5383552331c3f.json b/backend/.sqlx/query-5bd410d777a7a6d48129e9fee8402455082e0a172948be6441e5383552331c3f.json new file mode 100644 index 0000000000..08008e5900 --- /dev/null +++ b/backend/.sqlx/query-5bd410d777a7a6d48129e9fee8402455082e0a172948be6441e5383552331c3f.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT super_admin, devops FROM password WHERE email = $1 AND disabled = false FOR UPDATE", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "super_admin", + "type_info": "Bool" + }, + { + "ordinal": 1, + "name": "devops", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "5bd410d777a7a6d48129e9fee8402455082e0a172948be6441e5383552331c3f" +} diff --git a/backend/.sqlx/query-5ca0afc5a7b0437de221c8cc7b31e015ec604eb92a40b8649d054cabef1d8060.json b/backend/.sqlx/query-5ca0afc5a7b0437de221c8cc7b31e015ec604eb92a40b8649d054cabef1d8060.json new file mode 100644 index 0000000000..c1d11bffe1 --- /dev/null +++ b/backend/.sqlx/query-5ca0afc5a7b0437de221c8cc7b31e015ec604eb92a40b8649d054cabef1d8060.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(SELECT 1 FROM cloud_trial_offer WHERE email = $1 AND consumed_at IS NULL)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "5ca0afc5a7b0437de221c8cc7b31e015ec604eb92a40b8649d054cabef1d8060" +} diff --git a/backend/.sqlx/query-64bc01a5d88680febabd794b6472b6d22720d71e555501011e1d6b3418064ed0.json b/backend/.sqlx/query-64bc01a5d88680febabd794b6472b6d22720d71e555501011e1d6b3418064ed0.json new file mode 100644 index 0000000000..1eb8c87315 --- /dev/null +++ b/backend/.sqlx/query-64bc01a5d88680febabd794b6472b6d22720d71e555501011e1d6b3418064ed0.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO cloud_onboarding_profile (email, profile, created_by) VALUES ($1, $2, $3)\n ON CONFLICT (email) DO UPDATE SET profile = EXCLUDED.profile", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Jsonb", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "64bc01a5d88680febabd794b6472b6d22720d71e555501011e1d6b3418064ed0" +} diff --git a/backend/.sqlx/query-754598696e57a8c3ee6477d4f55f62e12019aa9582f06b9a47f7d04715eee24c.json b/backend/.sqlx/query-754598696e57a8c3ee6477d4f55f62e12019aa9582f06b9a47f7d04715eee24c.json new file mode 100644 index 0000000000..3136e1b433 --- /dev/null +++ b/backend/.sqlx/query-754598696e57a8c3ee6477d4f55f62e12019aa9582f06b9a47f7d04715eee24c.json @@ -0,0 +1,34 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE login_link SET consumed_at = now()\n WHERE token_hash = $1 AND consumed_at IS NULL AND expiration > now()\n RETURNING email, rd, created_by", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "rd", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "created_by", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Bpchar" + ] + }, + "nullable": [ + false, + true, + false + ] + }, + "hash": "754598696e57a8c3ee6477d4f55f62e12019aa9582f06b9a47f7d04715eee24c" +} diff --git a/backend/.sqlx/query-a2be5aeb7e663b0fe403726b4a41a7760b21c7edb651452c80b46b54ec964901.json b/backend/.sqlx/query-a2be5aeb7e663b0fe403726b4a41a7760b21c7edb651452c80b46b54ec964901.json new file mode 100644 index 0000000000..e6fd39901e --- /dev/null +++ b/backend/.sqlx/query-a2be5aeb7e663b0fe403726b4a41a7760b21c7edb651452c80b46b54ec964901.json @@ -0,0 +1,34 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT super_admin, devops, login_type FROM password WHERE email = $1 AND disabled = false", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "super_admin", + "type_info": "Bool" + }, + { + "ordinal": 1, + "name": "devops", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "login_type", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + false + ] + }, + "hash": "a2be5aeb7e663b0fe403726b4a41a7760b21c7edb651452c80b46b54ec964901" +} diff --git a/backend/.sqlx/query-ab16363a5225b022c7262f3caf5cd21ed1fbcc989fcc020c8246d5e2a313b72c.json b/backend/.sqlx/query-ab16363a5225b022c7262f3caf5cd21ed1fbcc989fcc020c8246d5e2a313b72c.json new file mode 100644 index 0000000000..ef7411614b --- /dev/null +++ b/backend/.sqlx/query-ab16363a5225b022c7262f3caf5cd21ed1fbcc989fcc020c8246d5e2a313b72c.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT consumed_at IS NOT NULL AS \"used!\" FROM login_link WHERE token_hash = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "used!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Bpchar" + ] + }, + "nullable": [ + null + ] + }, + "hash": "ab16363a5225b022c7262f3caf5cd21ed1fbcc989fcc020c8246d5e2a313b72c" +} diff --git a/backend/.sqlx/query-b2855a7bf20ec5a405d8c059e7b3b635507f545a514ff0d9142c663b779dd961.json b/backend/.sqlx/query-b2855a7bf20ec5a405d8c059e7b3b635507f545a514ff0d9142c663b779dd961.json new file mode 100644 index 0000000000..1bf06d5366 --- /dev/null +++ b/backend/.sqlx/query-b2855a7bf20ec5a405d8c059e7b3b635507f545a514ff0d9142c663b779dd961.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO cloud_trial_offer (email, created_by) VALUES ($1, $2)\n ON CONFLICT (email) DO NOTHING", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "b2855a7bf20ec5a405d8c059e7b3b635507f545a514ff0d9142c663b779dd961" +} diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 9f9388b41a..062211f925 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -a4da009a5eae72bd55f34de41ba7929b53d53c9b +1ba6fe83451f0a1f8fafe04b7187087d51e0f769 diff --git a/backend/migrations/20260827203157_login_link.down.sql b/backend/migrations/20260827203157_login_link.down.sql new file mode 100644 index 0000000000..aa26e1eee8 --- /dev/null +++ b/backend/migrations/20260827203157_login_link.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS login_link; diff --git a/backend/migrations/20260827203157_login_link.up.sql b/backend/migrations/20260827203157_login_link.up.sql new file mode 100644 index 0000000000..0be611f408 --- /dev/null +++ b/backend/migrations/20260827203157_login_link.up.sql @@ -0,0 +1,13 @@ +-- Single-use login links minted by a superadmin for one account. Consumed by an +-- unauthenticated GET that mints a session; the row is never a bearer credential itself. +CREATE TABLE login_link ( + token_hash CHAR(64) PRIMARY KEY, + email VARCHAR(255) NOT NULL REFERENCES password(email) ON DELETE CASCADE ON UPDATE CASCADE, + rd TEXT, + expiration TIMESTAMPTZ NOT NULL, + consumed_at TIMESTAMPTZ, + created_by VARCHAR(255) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE INDEX login_link_email_idx ON login_link (email); diff --git a/backend/migrations/20260828190901_cloud_trial_offer.down.sql b/backend/migrations/20260828190901_cloud_trial_offer.down.sql new file mode 100644 index 0000000000..0769b91752 --- /dev/null +++ b/backend/migrations/20260828190901_cloud_trial_offer.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS cloud_trial_offer; diff --git a/backend/migrations/20260828190901_cloud_trial_offer.up.sql b/backend/migrations/20260828190901_cloud_trial_offer.up.sql new file mode 100644 index 0000000000..d16ea6857c --- /dev/null +++ b/backend/migrations/20260828190901_cloud_trial_offer.up.sql @@ -0,0 +1,11 @@ +-- A pre-approved self-hosted Enterprise trial offered to an account created through a +-- pre-approved invite. No expiry: the offer lasts until a trial or subscription exists. +-- The cascade follows the account out on deletion and rename. A superadmin users-import +-- replaces every account by deleting and reinserting it, which takes these rows with it: +-- the offers, like the onboarding profiles, are recorded by the portal that minted them. +CREATE TABLE cloud_trial_offer ( + email VARCHAR(255) PRIMARY KEY REFERENCES password(email) ON DELETE CASCADE ON UPDATE CASCADE, + consumed_at TIMESTAMPTZ, + created_by VARCHAR(255) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); diff --git a/backend/migrations/20260909213500_cloud_onboarding_profile.down.sql b/backend/migrations/20260909213500_cloud_onboarding_profile.down.sql new file mode 100644 index 0000000000..3c67031c9f --- /dev/null +++ b/backend/migrations/20260909213500_cloud_onboarding_profile.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS cloud_onboarding_profile; diff --git a/backend/migrations/20260909213500_cloud_onboarding_profile.up.sql b/backend/migrations/20260909213500_cloud_onboarding_profile.up.sql new file mode 100644 index 0000000000..8e295ea6c6 --- /dev/null +++ b/backend/migrations/20260909213500_cloud_onboarding_profile.up.sql @@ -0,0 +1,9 @@ +-- Context an invite carried about the account's owner, written at provisioning and read by +-- onboarding to tailor itself (skip the source question it knows the answer to, later +-- template picks and starter prompts). Free-form JSON so new fields need no migration. +CREATE TABLE cloud_onboarding_profile ( + email VARCHAR(255) PRIMARY KEY REFERENCES password(email) ON DELETE CASCADE ON UPDATE CASCADE, + profile JSONB NOT NULL, + created_by VARCHAR(255) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 3558155af7..d18b23e1c0 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -1784,6 +1784,23 @@ pub async fn delete_expired_items(db: &DB) -> () { Err(e) => tracing::error!("Error deleting token: {}", e.to_string()), } + let expired_login_links_r: std::result::Result, _> = + // Expired rows stay a day so an open still reports "expired" rather than "invalid". + sqlx::query_scalar( + "DELETE FROM login_link WHERE expiration <= now() - interval '1 day' RETURNING token_hash", + ) + .fetch_all(db) + .await; + + match expired_login_links_r { + Ok(hashes) => { + if !hashes.is_empty() { + tracing::info!("deleted {} expired login links", hashes.len()) + } + } + Err(e) => tracing::error!("Error deleting login links: {}", e.to_string()), + } + let pip_resolution_r = sqlx::query_scalar!( "DELETE FROM pip_resolution_cache WHERE expiration <= now() RETURNING hash", ) diff --git a/backend/windmill-api-integration-tests/tests/login_link.rs b/backend/windmill-api-integration-tests/tests/login_link.rs new file mode 100644 index 0000000000..5c3a231a48 --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/login_link.rs @@ -0,0 +1,192 @@ +use serde_json::json; +use sqlx::{Pool, Postgres}; + +use windmill_test_utils::*; + +fn client() -> reqwest::Client { + reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build() + .unwrap() +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn login_link_is_single_use_and_same_origin(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api"); + let mint = |token: &'static str, body: serde_json::Value| { + client() + .post(format!("{base}/users/login_links")) + .header("Authorization", format!("Bearer {token}")) + .json(&body) + .send() + }; + + // Only a superadmin mints. + let resp = mint("SECRET_TOKEN_2", json!({"email": "test2@windmill.dev"})).await?; + assert_eq!(resp.status(), 401); + + // A superadmin account is never a valid target: the minting credential must not + // become an instance-wide role. + let resp = mint("SECRET_TOKEN", json!({"email": "test@windmill.dev"})).await?; + assert_eq!(resp.status(), 400); + + // An off-origin destination is refused before anything is minted. + let resp = mint( + "SECRET_TOKEN", + json!({"email": "test2@windmill.dev", "rd": "https://evil.example/"}), + ) + .await?; + assert_eq!(resp.status(), 400); + + let resp = mint( + "SECRET_TOKEN", + json!({"email": "test2@windmill.dev", "rd": "/user/workspaces?x=1"}), + ) + .await?; + assert_eq!(resp.status(), 201); + let link = resp.json::().await?; + let path = link["url"] + .as_str() + .unwrap() + .split_once("/api") + .unwrap() + .1 + .to_string(); + let consume_url = format!("{base}{path}"); + + // A promotion inside the link's window is re-checked at open time: no session, + // and the link is not spent while the account is privileged. + sqlx::query("UPDATE password SET super_admin = true WHERE email = 'test2@windmill.dev'") + .execute(&db) + .await?; + let resp = client().get(&consume_url).send().await?; + assert_eq!(resp.status(), 302); + assert_eq!( + resp.headers()["location"], + "/user/login_link_expired?reason=invalid" + ); + assert!(resp.headers().get("set-cookie").is_none()); + sqlx::query("UPDATE password SET super_admin = false WHERE email = 'test2@windmill.dev'") + .execute(&db) + .await?; + + // First open: session cookie for the target account, redirected to the stored rd. + let resp = client().get(&consume_url).send().await?; + assert_eq!(resp.status(), 302); + assert_eq!(resp.headers()["location"], "/user/workspaces?x=1"); + assert_eq!(resp.headers()["referrer-policy"], "no-referrer"); + let cookie = resp + .headers() + .get_all("set-cookie") + .iter() + .map(|c| c.to_str().unwrap().to_string()) + .find(|c| c.starts_with("token=")) + .expect("session cookie"); + assert!(cookie.contains("HttpOnly")); + let session = cookie + .split(';') + .next() + .unwrap() + .trim_start_matches("token=") + .to_string(); + let resp = client() + .get(format!("{base}/users/whoami")) + .header("Authorization", format!("Bearer {session}")) + .send() + .await?; + assert_eq!(resp.status(), 200); + assert_eq!( + resp.json::().await?["email"], + "test2@windmill.dev" + ); + + // Second open: burned, no cookie, bounced to the explanation page. + let resp = client().get(&consume_url).send().await?; + assert_eq!(resp.status(), 302); + assert_eq!( + resp.headers()["location"], + "/user/login_link_expired?reason=used" + ); + assert!(resp.headers().get("set-cookie").is_none()); + + Ok(()) +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn login_link_mint_can_require_a_login_type(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api"); + let mint = || { + client() + .post(format!("{base}/users/login_links")) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({"email": "test2@windmill.dev", "require_login_type": "pending_oauth"})) + .send() + }; + + // A password account is not the account the caller created: no link. + let resp = mint().await?; + assert_eq!(resp.status(), 409); + assert!(resp.text().await?.contains("login_type_mismatch")); + + sqlx::query( + "UPDATE password SET login_type = 'pending_oauth', password_hash = NULL WHERE email = 'test2@windmill.dev'", + ) + .execute(&db) + .await?; + let resp = mint().await?; + assert_eq!(resp.status(), 201); + Ok(()) +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn cloud_trial_offer_go_refuses_a_job_token(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + set_jwt_secret().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api"); + + // The answer is a signed-in portal login for the account, so the job-token check + // must come before every other gate: a script holding `$WM_TOKEN` is refused outright, + // where a browser session reaches the next check (off cloud, "no offer"). + let job_id = uuid::Uuid::new_v4(); + sqlx::query( + "INSERT INTO v2_job (id, workspace_id, created_by, permissioned_as, kind, tag, args) + VALUES ($1, 'test-workspace', 'test-user-2', 'u/test-user-2', 'script', 'deno', '{}'::jsonb)", + ) + .bind(job_id) + .execute(&db) + .await?; + let job_token = windmill_common::auth::create_token_for_owner( + &db, + "test-workspace", + "u/test-user-2", + "job", + 600, + "test2@windmill.dev", + &job_id, + None, + None, + ) + .await?; + + let go = |token: String| { + client() + .post(format!("{base}/users/cloud_trial_offer/go")) + .header("Authorization", format!("Bearer {token}")) + .send() + }; + let resp = go(job_token).await?; + assert_eq!(resp.status(), 403); + assert!(resp.text().await?.contains("job token")); + + let resp = go("SECRET_TOKEN_2".to_string()).await?; + assert_eq!(resp.status(), 404); + Ok(()) +} diff --git a/backend/windmill-api-users/src/users.rs b/backend/windmill-api-users/src/users.rs index 260df2f969..f152ba8808 100644 --- a/backend/windmill-api-users/src/users.rs +++ b/backend/windmill-api-users/src/users.rs @@ -46,7 +46,7 @@ use tracing::Instrument; use windmill_audit::audit_oss::audit_log; use windmill_audit::ActionKind; use windmill_common::audit::AuditAuthor; -use windmill_common::auth::{safe_token_prefix, TOKEN_PREFIX_LEN}; +use windmill_common::auth::{hash_token, safe_token_prefix, TOKEN_PREFIX_LEN}; use windmill_common::global_settings::AUTOMATE_USERNAME_CREATION_SETTING; use windmill_common::oauth2::InstanceEvent; use windmill_common::per_minute_counter::PerMinuteCounter; @@ -140,6 +140,16 @@ pub fn global_service() -> Router { ) .route("/tokens/list", get(list_tokens)) .route("/tokens/impersonate", post(impersonate)) + .route("/login_links", post(create_login_link)) + .route( + "/cloud_trial_offer", + post(set_cloud_trial_offer).get(get_cloud_trial_offer), + ) + .route("/cloud_trial_offer/go", post(go_cloud_trial_offer)) + .route( + "/onboarding_profile", + post(set_onboarding_profile).get(get_onboarding_profile), + ) .route("/usage", get(get_usage)) .route("/all_runnables", get(get_all_runnables)) .route("/refresh_token", get(refresh_token)) @@ -158,6 +168,7 @@ pub fn make_unauthed_service() -> Router { .route("/logout", post(logout).get(logout)) .route("/is_first_time_setup", get(is_first_time_setup)) .route("/request_password_reset", post(request_password_reset)) + .route("/login_link/{token}", get(consume_login_link)) .route("/is_smtp_configured", get(is_smtp_configured)) .route( "/is_password_login_disabled", @@ -255,11 +266,14 @@ pub struct WorkspaceInvite { #[derive(Deserialize)] pub struct NewUser { pub email: String, - pub password: String, + /// Required when `login_type` is `password` (the default), ignored otherwise. + pub password: Option, pub super_admin: bool, pub name: Option, pub company: Option, pub skip_email: Option, + /// `password`, `pending_oauth`, or a configured OAuth login client key. + pub login_type: Option, } #[derive(Deserialize)] @@ -3181,6 +3195,503 @@ async fn impersonate( Ok((StatusCode::CREATED, token)) } +const LOGIN_LINK_DEFAULT_TTL_S: u32 = 600; +const LOGIN_LINK_MAX_TTL_S: u32 = 900; +const LOGIN_LINK_DEFAULT_RD: &str = "/user/workspaces"; +const LOGIN_LINK_EXPIRED_PAGE: &str = "/user/login_link_expired"; + +#[derive(Deserialize)] +pub struct NewLoginLink { + pub email: String, + pub expires_in_s: Option, + pub rd: Option, + /// Refuse to mint unless the account still has this login type: a caller re-entering an + /// account it created can require `pending_oauth`, so the link stops working once the + /// owner has set a password or signed in with a provider. + pub require_login_type: Option, +} + +#[derive(Serialize)] +pub struct LoginLink { + pub url: String, + pub expires_at: chrono::DateTime, +} + +/// A post-login destination is only ever a same-origin path: anything else would hand the +/// fresh session's first navigation to another host. Control characters are refused because +/// browsers strip tab/newline from a `Location` before parsing it, so `/\t/host` reads as +/// the protocol-relative `//host`. +fn same_origin_rd(rd: Option) -> Option { + rd.filter(|r| { + r.starts_with('/') + && !r.starts_with("//") + && !r.contains('\\') + && !r.chars().any(|c| c.is_ascii_control()) + }) +} + +#[cfg(test)] +mod same_origin_rd_tests { + use super::same_origin_rd; + + fn accepts(rd: &str) -> bool { + same_origin_rd(Some(rd.to_string())).is_some() + } + + #[test] + fn only_plain_same_origin_paths_pass() { + assert!(accepts("/")); + assert!(accepts("/user/workspaces?rd=%2Fx")); + assert!(!accepts("https://evil.example/")); + assert!(!accepts("//evil.example/")); + assert!(!accepts("/\\evil.example/")); + assert!(!accepts("/\t/evil.example/")); + assert!(!accepts("/x\r\nSet-Cookie: a=b")); + assert!(!accepts("user/workspaces")); + } +} + +/// Both provisioning writes reference `password(email)`; a typo'd address from the +/// provisioning script should read as "no such account", not as a foreign-key error. +async fn require_account(tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, email: &str) -> Result<()> { + let exists = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM password WHERE email = $1)", + email + ) + .fetch_one(&mut **tx) + .await? + .unwrap_or(false); + if !exists { + return Err(Error::NotFound(format!("no account for {email}"))); + } + Ok(()) +} + +fn login_link_redirect(location: String) -> Response { + ( + StatusCode::FOUND, + [ + ("location", location), + ("referrer-policy", "no-referrer".to_string()), + ], + ) + .into_response() +} + +/// Mint a single-use link that signs `email` in when opened. The row is not a `token`: +/// it can only ever become a session, and burning it needs no cache invalidation. +async fn create_login_link( + Extension(db): Extension, + authed: ApiAuthed, + OptJobAuthed { job_id, .. }: OptJobAuthed, + Json(nl): Json, +) -> Result<(StatusCode, Json)> { + require_super_admin(&db, &authed).await?; + forbid_superadmin_job_token(&db, &authed.email, job_id).await?; + + let email = nl.email.to_lowercase(); + let rd = match nl.rd { + Some(rd) => Some(same_origin_rd(Some(rd)).ok_or_else(|| { + Error::BadRequest("rd must be a same-origin path starting with /".to_string()) + })?), + None => None, + }; + let ttl = nl + .expires_in_s + .unwrap_or(LOGIN_LINK_DEFAULT_TTL_S) + .clamp(1, LOGIN_LINK_MAX_TTL_S); + + let mut tx = db.begin().await?; + let target = sqlx::query!( + "SELECT super_admin, devops, login_type FROM password WHERE email = $1 AND disabled = false", + &email + ) + .fetch_optional(&mut *tx) + .await?; + let Some(target) = target else { + return Err(Error::NotFound(format!("no active account for {email}"))); + }; + // A link is a full session for its account; whoever holds the minting credential + // must not be able to turn it into an instance-wide role. + if target.super_admin || target.devops { + return Err(Error::BadRequest( + "login links cannot target superadmin or devops accounts".to_string(), + )); + } + if let Some(required) = nl.require_login_type.as_deref() { + if target.login_type != required { + return Err(Error::Generic( + StatusCode::CONFLICT, + format!( + "login_type_mismatch: {email} signs in with {}, not {required}", + target.login_type + ), + )); + } + } + + let token = rd_string(32); + let expires_at = chrono::Utc::now() + chrono::Duration::seconds(ttl as i64); + sqlx::query!( + "INSERT INTO login_link (token_hash, email, rd, expiration, created_by) + VALUES ($1, $2, $3, $4, $5)", + hash_token(&token), + &email, + rd, + expires_at, + &authed.email, + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "users.login_link.create", + ActionKind::Create, + "global", + Some(&email), + Some([("expires_in_s", &ttl.to_string()[..])].into()), + ) + .await?; + tx.commit().await?; + + let url = format!( + "{}/api/auth/login_link/{}", + (**BASE_URL.load()).clone(), + token + ); + Ok((StatusCode::CREATED, Json(LoginLink { url, expires_at }))) +} + +#[derive(Deserialize)] +pub struct CloudTrialOfferUpdate { + pub email: String, + #[serde(default)] + pub consumed: bool, +} + +#[derive(Deserialize)] +pub struct OnboardingProfileUpdate { + pub email: String, + pub profile: serde_json::Value, +} + +#[derive(Serialize)] +pub struct OnboardingProfile { + pub profile: Option, +} + +/// Context the invite carried about this account's owner, written at provisioning. +/// Onboarding tailors itself from it (today: `touch_point` answers the source question +/// so it is never asked); everything degrades to the plain flow when absent. +async fn set_onboarding_profile( + Extension(db): Extension, + authed: ApiAuthed, + OptJobAuthed { job_id, .. }: OptJobAuthed, + Json(body): Json, +) -> Result { + if !*CLOUD_HOSTED { + return Err(Error::NotFound("cloud only".to_string())); + } + require_super_admin(&db, &authed).await?; + forbid_superadmin_job_token(&db, &authed.email, job_id).await?; + if !body.profile.is_object() { + return Err(Error::BadRequest( + "profile must be a JSON object".to_string(), + )); + } + let email = body.email.to_lowercase(); + let mut tx = db.begin().await?; + require_account(&mut tx, &email).await?; + sqlx::query!( + "INSERT INTO cloud_onboarding_profile (email, profile, created_by) VALUES ($1, $2, $3) + ON CONFLICT (email) DO UPDATE SET profile = EXCLUDED.profile", + &email, + body.profile, + &authed.email + ) + .execute(&mut *tx) + .await?; + audit_log( + &mut *tx, + &authed, + "users.onboarding_profile.set", + ActionKind::Update, + "global", + Some(&email), + None, + ) + .await?; + tx.commit().await?; + Ok(format!("onboarding profile for {email} recorded")) +} + +async fn get_onboarding_profile( + Extension(db): Extension, + authed: ApiAuthed, +) -> JsonResult { + if !*CLOUD_HOSTED { + return Ok(Json(OnboardingProfile { profile: None })); + } + let profile = sqlx::query_scalar!( + "SELECT profile FROM cloud_onboarding_profile WHERE email = $1", + &authed.email + ) + .fetch_optional(&db) + .await?; + Ok(Json(OnboardingProfile { profile })) +} + +#[derive(Serialize)] +pub struct CloudTrialOffer { + pub offered: bool, +} + +/// What the customer portal answers when asked to sign a cloud account in and start its +/// pre-approved trial. +pub enum PortalTrialLogin { + /// Send the browser here: a short-lived portal login that starts the trial on landing. + LoginUrl(String), + /// The portal will not start one (a subscription exists, or it knows no offer); the + /// offer is spent and the browser goes to the portal home instead. + Unavailable { reason: String, portal_url: String }, +} + +async fn set_cloud_trial_offer( + Extension(db): Extension, + authed: ApiAuthed, + OptJobAuthed { job_id, .. }: OptJobAuthed, + Json(body): Json, +) -> Result { + if !*CLOUD_HOSTED { + return Err(Error::NotFound("cloud only".to_string())); + } + require_super_admin(&db, &authed).await?; + forbid_superadmin_job_token(&db, &authed.email, job_id).await?; + let email = body.email.to_lowercase(); + let mut tx = db.begin().await?; + require_account(&mut tx, &email).await?; + if body.consumed { + sqlx::query!( + "UPDATE cloud_trial_offer SET consumed_at = now() WHERE email = $1 AND consumed_at IS NULL", + &email + ) + .execute(&mut *tx) + .await?; + } else { + // A consumed offer stays consumed: a trial or subscription already exists for it. + sqlx::query!( + "INSERT INTO cloud_trial_offer (email, created_by) VALUES ($1, $2) + ON CONFLICT (email) DO NOTHING", + &email, + &authed.email + ) + .execute(&mut *tx) + .await?; + } + audit_log( + &mut *tx, + &authed, + "users.cloud_trial_offer.set", + ActionKind::Update, + "global", + Some(&email), + Some([("consumed", if body.consumed { "true" } else { "false" })].into()), + ) + .await?; + tx.commit().await?; + Ok(format!( + "cloud trial offer for {email} {}", + if body.consumed { + "consumed" + } else { + "recorded" + } + )) +} + +async fn offered(db: &DB, email: &str) -> Result { + Ok(sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM cloud_trial_offer WHERE email = $1 AND consumed_at IS NULL)", + email + ) + .fetch_one(db) + .await? + .unwrap_or(false)) +} + +async fn get_cloud_trial_offer( + Extension(db): Extension, + authed: ApiAuthed, +) -> JsonResult { + if !*CLOUD_HOSTED { + return Ok(Json(CloudTrialOffer { offered: false })); + } + Ok(Json(CloudTrialOffer { + offered: offered(&db, &authed.email).await?, + })) +} + +/// Where the browser goes to start the pre-approved trial: a signed-in portal login, or +/// the portal's front page with the reason it could not start one. +#[derive(Serialize)] +pub struct CloudTrialOfferGo { + pub location: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub reason: Option, +} + +/// The one click that turns a cloud account's pre-approved offer into a trial: the portal +/// is asked for a login that starts it, and the browser is handed over. The portal is the +/// authority on whether the offer still stands; its refusal spends the offer here so the +/// sidebar stops advertising it. +async fn go_cloud_trial_offer( + Extension(db): Extension, + authed: ApiAuthed, +) -> JsonResult { + // The answer carries a signed-in portal login for this account: a credential for + // another system, and asking for it starts the trial. A script running as the offered + // user holds their identity through `$WM_TOKEN`, so a job token must not be able to + // fetch it and hand it to whoever wrote the script. It is a POST answered as JSON, not + // a redirecting GET, so a cross-site top-level navigation cannot start the trial with + // the SameSite=Lax session cookie either; the frontend navigates to `location` itself. + if authed.job_id.is_some() { + return Err(Error::NotAuthorized( + "This endpoint cannot be called with a job token ($WM_TOKEN).".to_string(), + )); + } + if !*CLOUD_HOSTED || !offered(&db, &authed.email).await? { + return Err(Error::NotFound( + "no pre-approved trial offer for this account".to_string(), + )); + } + let outcome = crate::users_oss::portal_cloud_trial_login(&authed.email).await?; + let (location, reason) = match outcome { + PortalTrialLogin::LoginUrl(url) => (url, None), + PortalTrialLogin::Unavailable { reason, portal_url } => (portal_url, Some(reason)), + }; + let mut tx = db.begin().await?; + if reason.is_some() { + sqlx::query!( + "UPDATE cloud_trial_offer SET consumed_at = now() WHERE email = $1 AND consumed_at IS NULL", + &authed.email + ) + .execute(&mut *tx) + .await?; + } + audit_log( + &mut *tx, + &authed, + "users.cloud_trial_offer.go", + ActionKind::Execute, + "global", + Some(&authed.email), + Some([("outcome", reason.as_deref().unwrap_or("login"))].into()), + ) + .await?; + tx.commit().await?; + Ok(Json(CloudTrialOfferGo { location, reason })) +} + +#[derive(Deserialize)] +struct LoginLinkQuery { + rd: Option, +} + +async fn consume_login_link( + headers: axum::http::HeaderMap, + cookies: Cookies, + Extension(db): Extension, + Path(token): Path, + Query(query): Query, +) -> Result { + let bounce = |reason: &str| { + Ok(login_link_redirect(format!( + "{LOGIN_LINK_EXPIRED_PAGE}?reason={reason}" + ))) + }; + if token.len() != 32 { + return bounce("invalid"); + } + let t_hash = hash_token(&token); + // The account is unknown until the row is read, so only the global and per-IP tiers + // apply here; a 32-char random token leaves nothing for the per-account tier to guard. + windmill_common::login_rate_limit::check_and_increment_login_attempt( + &headers, + &t_hash[..TOKEN_PREFIX_LEN], + )?; + + let mut tx = db.begin().await?; + let link = sqlx::query!( + "UPDATE login_link SET consumed_at = now() + WHERE token_hash = $1 AND consumed_at IS NULL AND expiration > now() + RETURNING email, rd, created_by", + &t_hash + ) + .fetch_optional(&mut *tx) + .await?; + let Some(link) = link else { + let used = sqlx::query_scalar!( + "SELECT consumed_at IS NOT NULL AS \"used!\" FROM login_link WHERE token_hash = $1", + &t_hash + ) + .fetch_optional(&mut *tx) + .await?; + return bounce(match used { + Some(true) => "used", + Some(false) => "expired", + None => "invalid", + }); + }; + + // Re-checked at open time and locked through session creation: a promotion inside + // the link's window must not turn a link minted for an ordinary account into a + // privileged session. The bounce drops the transaction, so the link is not spent. + let target = sqlx::query!( + "SELECT super_admin, devops FROM password WHERE email = $1 AND disabled = false FOR UPDATE", + &link.email + ) + .fetch_optional(&mut *tx) + .await?; + let Some(target) = target else { + return bounce("invalid"); + }; + if target.super_admin || target.devops { + return bounce("invalid"); + } + + let session = create_session_token(&link.email, false, None, false, &mut tx, cookies).await?; + audit_log( + &mut *tx, + &AuditAuthor { + email: link.email.clone(), + username: link.email.clone(), + username_override: None, + token_prefix: Some(safe_token_prefix(&session)), + }, + "users.login", + ActionKind::Create, + "global", + Some(&truncate_token(&session)), + Some( + [ + ("method", "login_link"), + ("minted_by", link.created_by.as_str()), + ] + .into(), + ), + ) + .await?; + tx.commit().await?; + + let rd = link + .rd + .or_else(|| same_origin_rd(query.rd)) + .unwrap_or_else(|| LOGIN_LINK_DEFAULT_RD.to_string()); + Ok(login_link_redirect(rd)) +} + #[derive(Deserialize)] pub struct ImpersonateServiceAccountRequest { pub username: String, @@ -3571,12 +4082,63 @@ async fn get_all_runnables( #[derive(Deserialize, Debug, Clone)] pub struct LoginUserInfo { pub email: Option, + /// OIDC `email_verified` claim where the provider sends one. + #[serde(default, deserialize_with = "deserialize_lenient_bool")] + pub email_verified: Option, pub name: Option, pub company: Option, pub preferred_username: Option, pub displayName: Option, } +/// Some providers (Cognito among them) send `email_verified` as the strings "true"/"false"; +/// a strict bool would reject their whole userinfo document and break login. +fn deserialize_lenient_bool<'de, D: serde::Deserializer<'de>>( + d: D, +) -> std::result::Result, D::Error> { + Ok(match Option::::deserialize(d)? { + Some(serde_json::Value::Bool(b)) => Some(b), + Some(serde_json::Value::String(s)) => match s.trim().to_ascii_lowercase().as_str() { + "true" => Some(true), + "false" => Some(false), + _ => None, + }, + _ => None, + }) +} + +#[cfg(test)] +mod login_user_info_tests { + use super::LoginUserInfo; + + fn email_verified(json: &str) -> Option { + serde_json::from_str::(json) + .unwrap() + .email_verified + } + + #[test] + fn email_verified_accepts_bool_and_stringified_bool() { + assert_eq!( + email_verified(r#"{"email":"a@b","email_verified":true}"#), + Some(true) + ); + assert_eq!( + email_verified(r#"{"email":"a@b","email_verified":"true"}"#), + Some(true) + ); + assert_eq!( + email_verified(r#"{"email":"a@b","email_verified":"false"}"#), + Some(false) + ); + assert_eq!( + email_verified(r#"{"email":"a@b","email_verified":"maybe"}"#), + None + ); + assert_eq!(email_verified(r#"{"email":"a@b"}"#), None); + } +} + #[derive(Serialize)] struct InstanceUsernameInfo { username: String, diff --git a/backend/windmill-api-users/src/users_oss.rs b/backend/windmill-api-users/src/users_oss.rs index a42cce8405..a1eedd8563 100644 --- a/backend/windmill-api-users/src/users_oss.rs +++ b/backend/windmill-api-users/src/users_oss.rs @@ -26,3 +26,13 @@ pub async fn impersonate_service_account( "Service accounts require Windmill Enterprise Edition".to_string(), )) } + +#[cfg(not(feature = "private"))] +pub(crate) async fn portal_cloud_trial_login( + _email: &str, +) -> windmill_common::error::Result { + Err(windmill_common::error::Error::FeatureUnavailable( + "Starting a pre-approved trial from Windmill Cloud requires Windmill Enterprise Edition" + .to_string(), + )) +} diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index ba4eddd97e..74c487dca1 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -406,6 +406,28 @@ paths: "400": description: SMTP not configured + /auth/login_link/{token}: + get: + security: [] + summary: consume a single-use login link, set the session cookie and redirect + operationId: consumeLoginLink + tags: + - user + parameters: + - name: token + in: path + required: true + schema: + type: string + - name: rd + in: query + required: false + schema: + type: string + responses: + "302": + description: redirected to the post-login destination, or to /user/login_link_expired when the link is used, expired or unknown + /auth/reset_password: post: security: [] @@ -623,9 +645,14 @@ paths: skip_email: type: boolean description: Skip sending email notifications to the user + login_type: + type: string + description: >- + password (default, requires `password`), pending_oauth (no credential + until the first OAuth login proving the address adopts the account), or + a configured OAuth login client key required: - email - - password - super_admin responses: "201": @@ -6384,6 +6411,172 @@ paths: schema: type: string + /users/login_links: + post: + summary: mint a single-use login link for an account (require superadmin) + operationId: createLoginLink + tags: + - user + requestBody: + description: target account and link options + required: true + content: + application/json: + schema: + type: object + required: + - email + properties: + email: + type: string + expires_in_s: + type: integer + description: link lifetime in seconds, at most 900 (default 600) + rd: + type: string + description: same-origin path the browser lands on after login (default /user/workspaces) + require_login_type: + type: string + description: >- + mint only while the account still has this login type (for example + pending_oauth), so a link stops working once the owner has set a password + or signed in with a provider + responses: + "201": + description: login link minted + content: + application/json: + schema: + type: object + required: + - url + - expires_at + properties: + url: + type: string + expires_at: + type: string + format: date-time + "409": + description: the account does not have the required login type + + /users/cloud_trial_offer: + post: + summary: record or consume a pre-approved self-hosted trial offer for a cloud account (require superadmin, cloud only) + operationId: setCloudTrialOffer + tags: + - user + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - email + properties: + email: + type: string + consumed: + type: boolean + description: mark the offer used (a trial or subscription now exists) instead of recording it + responses: + "200": + description: offer recorded or consumed + content: + text/plain: + schema: + type: string + get: + summary: whether the signed-in account holds an unconsumed pre-approved self-hosted trial offer (cloud only) + operationId: getCloudTrialOffer + tags: + - user + responses: + "200": + description: offer state + content: + application/json: + schema: + type: object + required: + - offered + properties: + offered: + type: boolean + + /users/cloud_trial_offer/go: + post: + summary: start the signed-in account's pre-approved self-hosted trial on the customer portal (cloud only). A POST answered as JSON rather than a redirecting GET, so a cross-site navigation cannot start it; the browser navigates to `location` itself + operationId: goCloudTrialOffer + tags: + - user + responses: + "200": + description: where to go — the customer portal signed in with the trial being started, or the portal home with the reason the offer could not be used + content: + application/json: + schema: + type: object + required: + - location + properties: + location: + type: string + reason: + type: string + description: present when the portal refused (e.g. the account already has a subscription); the offer is then spent + "403": + description: called with a job token + "404": + description: no offer for this account + + /users/onboarding_profile: + post: + summary: record the invite context an account's onboarding tailors itself from (require superadmin, cloud only) + operationId: setOnboardingProfile + tags: + - user + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - email + - profile + properties: + email: + type: string + profile: + type: object + description: free-form context from the invite, every key optional. The frontend reads `touch_point` (answers onboarding's source question), `company` and `workspace_name` (prefill the first workspace's name), `hub_projects` (slugs surfaced first on an empty workspace), `tools` (integrations, used to pick hub projects when none are named) and `starter_prompts` (`[{label, prompt}]`, replacing the home page's example prompts); unknown keys are kept and ignored + responses: + "200": + description: profile recorded + content: + text/plain: + schema: + type: string + get: + summary: the invite context recorded for the signed-in account, if any (cloud only) + operationId: getOnboardingProfile + tags: + - user + responses: + "200": + description: the profile, or null when none was recorded + content: + application/json: + schema: + type: object + properties: + profile: + type: object + nullable: true + additionalProperties: true + /users/tokens/delete/{token_prefix}: delete: summary: delete token @@ -33702,7 +33895,7 @@ components: type: string login_type: type: string - enum: ["password", "github", "service_account"] + enum: ["password", "github", "service_account", "pending_oauth"] super_admin: type: boolean devops: diff --git a/backend/windmill-common/src/users.rs b/backend/windmill-common/src/users.rs index 1a49605e59..941328a8b2 100644 --- a/backend/windmill-common/src/users.rs +++ b/backend/windmill-common/src/users.rs @@ -21,6 +21,13 @@ pub const SUPERADMIN_SYNC_EMAIL: &str = "superadmin_sync@windmill.dev"; pub const COOKIE_NAME: &str = "token"; +/// `password.login_type` of an account created for someone before they have signed in: +/// no credential of its own (password login and reset require `'password'`), reachable +/// only through a superadmin-minted login link until either `set_password` turns it into +/// a password account or the first OAuth login proving the same address adopts it and +/// rewrites `login_type` to the provider. +pub const PENDING_OAUTH_LOGIN_TYPE: &str = "pending_oauth"; + /// Prefix for user-based permissioned_as values: "u/" pub const PERMISSIONED_AS_USER_PREFIX: &str = "u/"; /// Prefix for group-based permissioned_as values: "g/" diff --git a/docs/auth-surface.md b/docs/auth-surface.md new file mode 100644 index 0000000000..0876a1184a --- /dev/null +++ b/docs/auth-surface.md @@ -0,0 +1,50 @@ +# Auth surface: facts that are easy to get wrong + +Symbols, not line numbers, are cited: they drift less. + +- **Credential precedence** (`windmill-api-auth/src/auth.rs` `extract_token`): `Authorization: Bearer` + → `token` cookie → `?token=` query param. A URL with `?token=` is a credential on every route, but + an existing cookie silently wins over it. +- **`AUTH_CACHE`** caches a token's identity for 120 s. Deleting a token row does not purge it: the + DB trigger (`migrations/20260316000001_token_hash_pk_swap.up.sql`) notifies only for + `label = 'session'` rows, and `delete_token` never calls `invalidate_token_from_cache`. +- **Sessions** are `token` rows with `label='session'` plus the HttpOnly `token` cookie, minted only + by `create_session_token` (`windmill-api-users/src/users.rs`). `GET /api/users/refresh_token` + mints one for any non-job token but returns plain text, no redirect. +- **`tokens/impersonate`** (superadmin) returns a multi-use token and sets no cookie. +- **Every superadmin route refuses a job token**: `require_super_admin` + (`windmill-api-auth/src/lib.rs`) errors on `authed.job_id.is_some()`. A script that needs + `users/create`, `tokens/impersonate`, `set_login_type`, … must use a dedicated superadmin user + token stored as a secret, never `$WM_TOKEN`. Token scopes cannot narrow superadmin routes. +- **`login_type`** (`password` table) is a free-form `VARCHAR(50)`. Password login and password + reset require `login_type = 'password'`; `set_password` also accepts `pending_oauth` and turns + the account into a `password` one in the same statement (an account created ahead of its owner + gets its first credential that way, or through the OAuth claim below). +- **Login links** (`login_link` table, `POST /users/login_links` superadmin-only, + `GET /auth/login_link/{token}` unauthenticated): single-use, ≤15 min, a session cookie and a + 302 to a same-origin `rd`. `require_login_type` on the mint refuses (409) an account whose + `login_type` has moved on — the way a caller re-entering an account it created stops being + able to once the owner has a password or a provider. +- **Pre-approved trial offer** (`cloud_trial_offer`, cloud-only routes under + `/users/cloud_trial_offer`): written by a superadmin at provisioning, consumed by + `{consumed: true}` or by the portal's refusal; `…/go` is the one Windmill→portal hop that + mints a portal login, over the same `CUSTOMER_SERVICE_TOKEN` trust the onboarding hook uses + (`users_ee.rs`, the portal's admin token). It never expires on its own. +- **OAuth login** (`oauth2_ee.rs` `login_externally`, decision in `existing_login_decision`) + matches an existing account by lowercased email only. Same provider → login; a + `pending_oauth` account (see `PENDING_OAUTH_LOGIN_TYPE`) is **claimed** by the first login + whose address the provider itself asserted and did not mark unverified — `login_type` becomes + the client key and the hash is nulled; otherwise `require_preexisting_user_for_oauth` decides: + on, *every* existing account is loggable-into by any provider; off, "exists but with a + different login type". A new account gets `login_type = `. +- **OAuth email trust**: `LoginUserInfo.email_verified` is read leniently (bool or + "true"/"false" strings) and is only consulted for the claim above; only GitHub is filtered to + `primary && verified`; a missing email is fabricated from `name` as `@windmill.dev` and + reaches `login_externally` with `email_asserted = false`. +- **`GET /api/oauth/login/{client}`** is an unauthenticated 302 to the provider — a plain link + from any page starts SSO. +- **`CLOUD_HOSTED`** is presence-tested (`windmill-common/src/worker.rs`): `CLOUD_HOSTED=false` + still enables cloud mode. Of the routes above only the cloud trial offer and onboarding + profile routes are cloud-gated; for the rest, cloud only adds quotas. +- **`CREATE_WORKSPACE_REQUIRE_SUPERADMIN`** defaults to `true` when unset; only the literal + `"true"` enables it when set. diff --git a/docs/feature-telemetry.md b/docs/feature-telemetry.md index ac64800c36..cbe130a004 100644 --- a/docs/feature-telemetry.md +++ b/docs/feature-telemetry.md @@ -4,10 +4,10 @@ anonymous usage-stats payload. It answers "does anyone use this, and which variant do they pick" without any identifying data leaving the instance. -It currently carries 49 registered actions across eighteen features (`ai_session`, `ai_chat`, +It currently carries 50 registered actions across nineteen features (`ai_session`, `ai_chat`, `ai_fix`, `ai_agent`, `ai_agent_eval`, `app_sandbox`, `datatable`, `flow_editor`, `flow_run`, `flow_step`, `home`, `run_form`, `debugger`, `trigger`, `command_script`, `hub_script`, -`usage_meter`, `sso_groups_claim`). Nearly all of the +`usage_meter`, `sso_groups_claim`, `cloud_trial_offer`). Nearly all of the product is uninstrumented, so new user-facing work is the opportunity to change that. ## When to instrument diff --git a/frontend/src/lib/cloud.ts b/frontend/src/lib/cloud.ts index dbf4a58f48..30b2e54b75 100644 --- a/frontend/src/lib/cloud.ts +++ b/frontend/src/lib/cloud.ts @@ -6,6 +6,15 @@ export function isCloudHosted(): boolean { // may be missing or a stub with no `location`. Same defensive shape as // `isChromiumBrowser`. if (!BROWSER) return false + // Dev only: the cloud-specific UI (quotas, plan upgrade, the pre-approved trial offer) + // is otherwise unreachable from localhost. `localStorage.cloudHostedOverride = '1'` opts + // a browser in against a backend started with CLOUD_HOSTED. + if ( + import.meta.env.DEV && + globalThis.window?.localStorage?.getItem('cloudHostedOverride') === '1' + ) { + return true + } return globalThis.window?.location?.hostname == 'app.windmill.dev' } diff --git a/frontend/src/lib/components/InstanceSettings.svelte b/frontend/src/lib/components/InstanceSettings.svelte index f965362659..db9fc294cc 100644 --- a/frontend/src/lib/components/InstanceSettings.svelte +++ b/frontend/src/lib/components/InstanceSettings.svelte @@ -1086,8 +1086,8 @@ the flow editor, which skin approval steps are given, how data tables and their migrations are set up and used, how often an empty workspace home is seen, how often the home page’s create menu and hub-project picker are opened and from which entry - point, and the name of any public hub project imported from the home page and how far - that import got, last 30 days)
  • feature adoption (counts of which flow, script, trigger, worker and data table @@ -1150,8 +1150,8 @@ the flow editor, which skin approval steps are given, how data tables and their migrations are set up and used, how often an empty workspace home is seen, how often the home page’s create menu and hub-project picker are opened and from which entry - point, and the name of any public hub project imported from the home page and how far - that import got, last 30 days)
  • feature adoption (counts of which flow, script, trigger, worker and data table diff --git a/frontend/src/lib/components/Login.svelte b/frontend/src/lib/components/Login.svelte index e4b73478aa..da032717f6 100644 --- a/frontend/src/lib/components/Login.svelte +++ b/frontend/src/lib/components/Login.svelte @@ -1,4 +1,5 @@ + +
    + (accountSetup.open = true)} + /> + {#if isCollapsed} + + + + + + {/if} +
    diff --git a/frontend/src/lib/components/sidebar/FinishAccountSetup.svelte b/frontend/src/lib/components/sidebar/FinishAccountSetup.svelte new file mode 100644 index 0000000000..c565ccfad2 --- /dev/null +++ b/frontend/src/lib/components/sidebar/FinishAccountSetup.svelte @@ -0,0 +1,173 @@ + + + +
    +

    + Your account {email} was created from an invite and + has no sign-in method of its own yet. Pick one so you can come back any time. +

    + + {#if logins.length === 0 && !saml && !passwordAllowed} +

    + No sign-in method is available on this instance right now; ask an administrator. +

    + {/if} + {#if logins.length > 0 || saml} +
    + Sign in with a provider +
    + {#each logins as login (login.type)} + {@const Icon = icons[login.type]} + + {/each} + {#if saml} + + {/if} +
    +

    + Sign in to the provider as {email}; a different address is refused and you stay signed in + here. +

    +
    + {#if passwordAllowed} +
    +
    + or +
    +
    + {/if} + {/if} + + {#if passwordAllowed} +
    + Set a password +
    + + +
    +

    A password account keeps signing in with the password only.

    +
    + {/if} +
    +     diff --git a/frontend/src/lib/components/sidebar/MenuButton.svelte b/frontend/src/lib/components/sidebar/MenuButton.svelte index 71d88b071b..3d0286a4af 100644 --- a/frontend/src/lib/components/sidebar/MenuButton.svelte +++ b/frontend/src/lib/components/sidebar/MenuButton.svelte @@ -51,6 +51,9 @@ // Accessible name when the visible label is absent or only shown some of // the time, so the button stays announceable in every state. ariaLabel?: string | undefined + // Classes for the label line only — `class` reaches the button, the label and the + // sublabel alike, which is the wrong tool for colouring one line of the two. + labelClass?: string | undefined } let { @@ -75,7 +78,8 @@ showChevron = false, emphasizeLabel = false, disableTitle = false, - ariaLabel = undefined + ariaLabel = undefined, + labelClass = undefined }: Props = $props() let buttonRef: HTMLButtonElement | HTMLAnchorElement | undefined = $state(undefined) @@ -161,7 +165,8 @@ 'whitespace-pre truncate w-full', emphasizeLabel ? 'text-primary text-sm font-semibold' : sidebarClasses.text, 'transition-all', - classNames + classNames, + labelClass )} title={disableTitle ? undefined : label} > diff --git a/frontend/src/lib/components/sidebar/SettingsMenu.svelte b/frontend/src/lib/components/sidebar/SettingsMenu.svelte index b7a3a363bb..857e422bdd 100644 --- a/frontend/src/lib/components/sidebar/SettingsMenu.svelte +++ b/frontend/src/lib/components/sidebar/SettingsMenu.svelte @@ -17,7 +17,8 @@ Newspaper, Crown, Gauge, - Trash2 + Trash2, + KeyRound } from 'lucide-svelte' import { base } from '$app/paths' import { goto } from '$lib/navigation' @@ -35,6 +36,7 @@ import SideBarNotification from './SideBarNotification.svelte' import { markChangelogsOpened, readRecentChangelogs } from './changelogs' import { USER_SETTINGS_HASH, SUPERADMIN_SETTINGS_HASH } from './settings' + import { accountSetup } from './accountSetup.svelte' import { EXECUTIONS_HINT } from './executionsHint' import { userWorkspaces, @@ -207,6 +209,10 @@ : []) ]) + // An account entered through an invite link that still has no credentials of its own; + // the entry (and the sidebar banner it echoes) disappears once it does. + let pendingSetup = $derived(accountSetup.pending) + const items = $derived([ { displayName: 'Help', @@ -226,6 +232,17 @@ : ($userStore?.email ?? 'User'), icon: $userStore?.is_admin || $userStore?.non_member ? Crown : User, submenuItems: [ + ...(pendingSetup + ? [ + { + displayName: 'Finish account setup', + icon: KeyRound, + // The dropdown closes on this click; the modal opens once it is gone so its own + // buttons don't compete with the menu's outside-click handling. + action: () => setTimeout(() => (accountSetup.open = true), 50) + } + ] + : []), { displayName: 'Account settings', icon: Settings, diff --git a/frontend/src/lib/components/sidebar/SidebarUsage.svelte b/frontend/src/lib/components/sidebar/SidebarUsage.svelte index 1625ed83e2..6ecf81fa75 100644 --- a/frontend/src/lib/components/sidebar/SidebarUsage.svelte +++ b/frontend/src/lib/components/sidebar/SidebarUsage.svelte @@ -1,8 +1,18 @@ + + @@ -902,6 +911,18 @@ /> {/snippet} + +{#snippet accountSetupBanner(collapsed: boolean)} + {#if accountSetup.pending} +
    + +
    + {/if} +{/snippet} + {#snippet brandMark(collapsed: boolean)} @@ -929,6 +950,13 @@ {/snippet} +{#if accountSetup.pending} + accountSetup.refresh()} + /> +{/if} {#if page.status == 404} @@ -1101,6 +1129,7 @@ {/if}
    + {@render accountSetupBanner(false)}
    @@ -1239,6 +1268,7 @@ {/if}
    + {@render accountSetupBanner(isCollapsed)}
    diff --git a/frontend/src/routes/(root)/(logged)/user/(user)/onboarding/+page.svelte b/frontend/src/routes/(root)/(logged)/user/(user)/onboarding/+page.svelte index 34a9534117..035dbfd3b7 100644 --- a/frontend/src/routes/(root)/(logged)/user/(user)/onboarding/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/user/(user)/onboarding/+page.svelte @@ -25,6 +25,7 @@ MessageCircleCode } from 'lucide-svelte' import { sendUserToast } from '$lib/toast' + import { onboardingProfile } from '$lib/onboardingProfile' // Define step names as constants for better maintainability const STEP_SOURCE = 'source' @@ -51,6 +52,25 @@ // The survey was skipped, so the last step has nothing to go back to. let skippedSurvey = $state(false) + // An invited account arrives with the survey already answered: the invite that brought + // them here is how they heard about us, and their use case was researched before it was + // sent. Neither question is asked; the known source is recorded and they go straight to + // naming their workspace. Resolved before first paint: rendering a survey step and + // yanking it away a frame later reads as a glitch. + let invitedTouchPoint = $state(null) + let profileReady = $state(false) + async function loadInviteProfile() { + const profile = await onboardingProfile() + if (profile?.touch_point) { + invitedTouchPoint = profile.touch_point + // An account that already has somewhere to go leaves from here; painting the + // survey behind that navigation would show a step this account never takes. + if (await skip()) return + } + profileReady = true + } + loadInviteProfile() + async function loadWorkspaceStep() { try { const [workspaces, invites] = await Promise.all([ @@ -172,30 +192,36 @@ } } - async function skip() { + /** Declines the survey; true when that left onboarding altogether. */ + async function skip(): Promise { isSubmitting = true try { + // The known source still counts when the rest of the survey is declined. await UserService.submitOnboardingData({ - requestBody: {} + requestBody: invitedTouchPoint ? { touch_point: invitedTouchPoint } : {} }) } catch (error) { console.error('Error skipping onboarding:', error) - } finally { - await workspaceStepReady - isSubmitting = false - // Skipping the survey is not skipping naming the workspace: the questions are ours, - // the workspace is theirs. - skippedSurvey = true - if (alreadyPlaced) { - leaveOnboarding() - } else { - currentStep = STEP_WORKSPACE - } } + await workspaceStepReady + isSubmitting = false + // Skipping the survey is not skipping naming the workspace: the questions are ours, + // the workspace is theirs. + skippedSurvey = true + if (alreadyPlaced) { + await leaveOnboarding() + return true + } + currentStep = STEP_WORKSPACE + return false } -{#if currentStep === STEP_SOURCE} +{#if !profileReady} + + +{:else if currentStep === STEP_SOURCE}
    @@ -328,13 +354,16 @@ {/snippet} -
    -
    -
    -
    -
    + {#if !invitedTouchPoint} + +
    +
    +
    +
    +
    +
    -
    + {/if}
    {/if} diff --git a/frontend/src/routes/(root)/+layout.svelte b/frontend/src/routes/(root)/+layout.svelte index 4de44442a2..369582e7a2 100644 --- a/frontend/src/routes/(root)/+layout.svelte +++ b/frontend/src/routes/(root)/+layout.svelte @@ -4,6 +4,7 @@ import { page } from '$app/state' import { UserService, WorkspaceService } from '$lib/gen' import { logoutWithRedirect } from '$lib/logoutKit' + import { noteSessionEmail } from '$lib/onboardingProfile' import { clearWorkspaceFromStorage, userStore, @@ -161,6 +162,7 @@ ) } let user = await UserService.globalWhoami() + noteSessionEmail(user.email) console.log(`Welcome back ${user.email}`) } } catch (e) { diff --git a/frontend/src/routes/user/login_callback/[client_name]/+page.svelte b/frontend/src/routes/user/login_callback/[client_name]/+page.svelte index e45b43b871..b1e0f263a3 100644 --- a/frontend/src/routes/user/login_callback/[client_name]/+page.svelte +++ b/frontend/src/routes/user/login_callback/[client_name]/+page.svelte @@ -29,7 +29,29 @@ const rd = rawRd?.startsWith('http') && !isValidLogoutRedirect(rawRd) ? null : rawRd const closeUponLogin = getCookie('close') == 'true' || localStorage.getItem('closeUponLogin') == 'true' + // "Finish account setup" sent a signed-in account with no credentials of its own to a + // provider. Whatever went wrong on the way back — the consent screen cancelled, an + // address mismatch, an unverified address, a domain rule — that session is the only + // way into the account, so it must survive: report and go home rather than log out. + // Read before the backend call, which clears the cookie whether or not it adopts. + // SAML's ACS answers a top-level POST from the IdP, so a refusal there arrives here + // as a redirect with the flag in the query. + const finishingSetup = + !!getCookie('finish_setup') || page.url.searchParams.get('finish_setup') === '1' + function backToSetup(message: string) { + document.cookie = 'finish_setup=; path=/; max-age=0; SameSite=Lax' + sendUserToast(message, true) + goto('/') + } if (error) { + if (finishingSetup) { + backToSetup( + error.includes('finish_setup_mismatch') + ? error.replace(/^.*finish_setup_mismatch:\s*/, '') + : `Signing in with ${clientName} did not go through (${error}). Your account is unchanged.` + ) + return + } sendUserToast(`Error trying to login with ${clientName} ${error}`, true) if (closeUponLogin) { closeUponLoginError(`Error trying to login with ${clientName} ${error}`) @@ -40,6 +62,11 @@ try { await UserService.loginWithOauth({ requestBody: { code, state }, clientName }) } catch (e) { + const message = String(e?.body ?? e?.message ?? '') + if (finishingSetup) { + backToSetup(message.replace(/^.*finish_setup_mismatch:\s*/, '')) + return + } if (closeUponLogin) { closeUponLoginError(e.body ?? e.message) return diff --git a/frontend/src/routes/user/login_link_expired/+page.svelte b/frontend/src/routes/user/login_link_expired/+page.svelte new file mode 100644 index 0000000000..b160985bb2 --- /dev/null +++ b/frontend/src/routes/user/login_link_expired/+page.svelte @@ -0,0 +1,23 @@ + + + + + From a95e950529f6f01a220e09d322d5a4fb507ea694 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 14 Sep 2026 22:06:56 +0200 Subject: [PATCH 10/44] feat(cli): list, get and restore trashed items with wmill trash (#11125) * feat(cli): list, get and restore trashed items from the CLI * docs(cli): tell agents a sync push deletion is restorable with wmill trash * refactor(cli): share the ApiError formatting and type trash flags as integers --- backend/windmill-api/openapi.yaml | 147 ++++++++++++++++++ cli/src/commands/sync/sync.ts | 2 +- cli/src/commands/trash/trash.ts | 147 ++++++++++++++++++ cli/src/guidance/core.ts | 2 + cli/src/guidance/skills.gen.ts | 21 +++ cli/src/main.ts | 15 +- cli/src/utils/utils.ts | 17 ++ cli/test/trash_commands.test.ts | 66 ++++++++ .../lib/components/settings/Trashbin.svelte | 2 +- frontend/src/lib/services/trashService.ts | 69 -------- .../auto-generated/cli/cli-commands.md | 21 +++ system_prompts/auto-generated/prompts.ts | 21 +++ .../skills/cli-commands/SKILL.md | 21 +++ 13 files changed, 471 insertions(+), 80 deletions(-) create mode 100644 cli/src/commands/trash/trash.ts create mode 100644 cli/test/trash_commands.test.ts delete mode 100644 frontend/src/lib/services/trashService.ts diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 74c487dca1..21c059bc1a 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -293,6 +293,108 @@ paths: items: $ref: "#/components/schemas/AuditLog" + /w/{workspace}/trash/list: + get: + summary: list the workspace trashbin (requires admin privilege) + operationId: listTrash + tags: + - trash + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: item_kind + in: query + description: > + only return items of this kind: script, flow, app, schedule, variable, + resource, or a trigger kind such as http_trigger + schema: + type: string + - name: page + in: query + description: which page to return (starts at 0, default 0) + schema: + type: integer + - name: per_page + in: query + description: number of items to return for a given page (default 100, max 1000) + schema: + type: integer + responses: + "200": + description: the trashed items, most recently deleted first + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/TrashItem" + + /w/{workspace}/trash/get/{id}: + get: + summary: get a trashed item with the data it was deleted with (requires admin privilege) + operationId: getTrashItem + tags: + - trash + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/PathId" + responses: + "200": + description: the trashed item + content: + application/json: + schema: + $ref: "#/components/schemas/TrashItemWithData" + + /w/{workspace}/trash/restore/{id}: + post: + summary: restore a trashed item to its path (requires admin privilege) + operationId: restoreTrashItem + tags: + - trash + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/PathId" + responses: + "200": + description: item restored + content: + text/plain: + schema: + type: string + + /w/{workspace}/trash/delete/{id}: + delete: + summary: permanently delete a trashed item (requires admin privilege) + operationId: permanentlyDeleteTrashItem + tags: + - trash + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/PathId" + responses: + "200": + description: item permanently deleted + content: + text/plain: + schema: + type: string + + /w/{workspace}/trash/empty: + post: + summary: permanently delete every item in the workspace trashbin (requires admin privilege) + operationId: emptyTrash + tags: + - trash + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: trashbin emptied + content: + text/plain: + schema: + type: string + /auth/login: post: security: [] @@ -30015,6 +30117,51 @@ components: - operation - action_kind + TrashItem: + type: object + properties: + id: + type: integer + format: int64 + workspace_id: + type: string + item_kind: + type: string + description: script, flow, app, schedule, variable, resource, or a trigger kind such as http_trigger + item_path: + type: string + deleted_by: + type: string + deleted_at: + type: string + format: date-time + expires_at: + type: string + format: date-time + description: when the item is permanently deleted unless restored first + required: + - id + - workspace_id + - item_kind + - item_path + - deleted_by + - deleted_at + - expires_at + + TrashItemWithData: + allOf: + - $ref: "#/components/schemas/TrashItem" + - type: object + properties: + item_data: + type: object + additionalProperties: true + description: > + the deleted rows as they were stored; the shape depends on the kind, and a + secret variable's value stays encrypted + required: + - item_data + MainArgSignature: type: object properties: diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index 5f67d194fe..287dd6ae12 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -6611,7 +6611,7 @@ export async function push( if (deletedSecretBearing.length > 0) { log.info( colors.gray( - `${describeSecretBearingChanges(deletedSecretBearing)} deleted. The workspace trashbin keeps a deleted item for three days; a workspace admin can restore it from Workspace settings -> Trashbin.`, + `${describeSecretBearingChanges(deletedSecretBearing)} deleted. The workspace trashbin keeps a deleted item for three days; a workspace admin can restore it with \`wmill trash list\` and \`wmill trash restore \`, or from Workspace settings -> Trashbin.`, ), ); } diff --git a/cli/src/commands/trash/trash.ts b/cli/src/commands/trash/trash.ts new file mode 100644 index 0000000000..1d571e2884 --- /dev/null +++ b/cli/src/commands/trash/trash.ts @@ -0,0 +1,147 @@ +import { GlobalOptions } from "../../types.ts"; +import { requireLogin } from "../../core/auth.ts"; +import { resolveWorkspace } from "../../core/context.ts"; +import { Command } from "@cliffy/command"; +import { Table } from "@cliffy/table"; +import { colors } from "@cliffy/ansi/colors"; +import * as log from "../../core/log.ts"; +import { mergeConfigWithConfigFile } from "../../core/conf.ts"; +import * as wmill from "../../../gen/services.gen.ts"; +import { apiErrorMessage, formatTimestamp } from "../../utils/utils.ts"; + +async function list( + opts: GlobalOptions & { + json?: boolean; + kind?: string; + page?: number; + limit?: number; + } +) { + if (opts.json) log.setSilent(true); + opts = await mergeConfigWithConfigFile(opts); + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + + if (opts.page !== undefined && opts.page < 1) { + throw new Error("--page starts at 1"); + } + + const items = await wmill.listTrash({ + workspace: workspace.workspaceId, + itemKind: opts.kind, + // The trash endpoint counts pages from 0, unlike the API's other list + // endpoints whose `page` starts at 1; the flag counts from 1 like those. + page: opts.page === undefined ? undefined : opts.page - 1, + perPage: opts.limit, + }); + + if (opts.json) { + console.log(JSON.stringify(items)); + return; + } + if (items.length === 0) { + log.info("No trashed items found."); + return; + } + new Table() + .header(["ID", "Kind", "Path", "Deleted by", "Deleted at", "Expires at"]) + .padding(2) + .border(true) + .body( + items.map((item) => [ + String(item.id), + item.item_kind, + item.item_path, + item.deleted_by, + formatTimestamp(item.deleted_at), + formatTimestamp(item.expires_at), + ]) + ) + .render(); + log.info( + colors.gray( + "`wmill trash get ` shows what an item held, `wmill trash restore ` puts it back." + ) + ); +} + +async function get(opts: GlobalOptions & { json?: boolean }, id: number) { + if (opts.json) log.setSilent(true); + opts = await mergeConfigWithConfigFile(opts); + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + + const item = await wmill.getTrashItem({ + workspace: workspace.workspaceId, + id, + }); + + if (opts.json) { + console.log(JSON.stringify(item)); + return; + } + console.log(colors.bold("ID:") + " " + item.id); + console.log(colors.bold("Kind:") + " " + item.item_kind); + console.log(colors.bold("Path:") + " " + item.item_path); + console.log(colors.bold("Deleted by:") + " " + item.deleted_by); + console.log(colors.bold("Deleted at:") + " " + formatTimestamp(item.deleted_at)); + console.log(colors.bold("Expires at:") + " " + formatTimestamp(item.expires_at)); + console.log(colors.bold("Data:")); + console.log(JSON.stringify(item.item_data, null, 2)); +} + +async function restore(opts: GlobalOptions, ...ids: number[]) { + opts = await mergeConfigWithConfigFile(opts); + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + + let failed = 0; + for (const id of ids) { + try { + const message = await wmill.restoreTrashItem({ + workspace: workspace.workspaceId, + id, + }); + log.info(colors.green(message)); + } catch (e) { + failed += 1; + log.error( + `Could not restore trash item ${id}: ${apiErrorMessage(e) ?? String(e)}` + ); + } + } + if (failed > 0) { + process.exitCode = 1; + } +} + +const command = new Command() + .description( + "List, inspect and restore items deleted in the last three days (requires admin)" + ) + .option("--json", "Output as JSON (for piping to jq)") + .option( + "--kind ", + "Only items of this kind: script, flow, app, schedule, variable, resource or a trigger kind such as http_trigger" + ) + .option("--limit ", "Number of items to return (default 100, max 1000)") + .option("--page ", "Page to return, starting at 1") + .action(list as any) + .command("list", "List trashed items, most recently deleted first") + .option("--json", "Output as JSON (for piping to jq)") + .option( + "--kind ", + "Only items of this kind: script, flow, app, schedule, variable, resource or a trigger kind such as http_trigger" + ) + .option("--limit ", "Number of items to return (default 100, max 1000)") + .option("--page ", "Page to return, starting at 1") + .action(list as any) + .command("get", "Show a trashed item and the data it was deleted with") + .arguments("") + .option("--json", "Output as JSON (for piping to jq)") + .action(get as any) + .command("restore", "Put trashed items back at their paths") + .arguments("") + .action(restore as any); + +export default command; diff --git a/cli/src/guidance/core.ts b/cli/src/guidance/core.ts index 2a9bb34fe0..202a6feaad 100644 --- a/cli/src/guidance/core.ts +++ b/cli/src/guidance/core.ts @@ -165,6 +165,8 @@ No CI workflow runs \`wmill sync push\` automatically, so deploy directly from t - \`wmill sync push --dry-run\` to preview. - \`wmill sync push\` to apply. +A push deletes remote items that have no local file. They land in the workspace trashbin for three days: \`wmill trash list\` shows them and \`wmill trash restore \` puts one back (both need a workspace admin). + ### In both cases Only deploy when the user explicitly asks to deploy, publish, push, or ship — not when they say "run", "try", or "test". For testing local edits use the per-entity \`preview\` commands (\`wmill script preview\`, \`wmill flow preview\`) — they don't deploy. diff --git a/cli/src/guidance/skills.gen.ts b/cli/src/guidance/skills.gen.ts index b8b71b1de9..3f63574b83 100644 --- a/cli/src/guidance/skills.gen.ts +++ b/cli/src/guidance/skills.gen.ts @@ -7720,6 +7720,27 @@ Manage API tokens - \`--expiration \` - Token expiration (ISO 8601 timestamp) - \`token delete \` - Delete a token by its prefix +### trash + +List, inspect and restore items deleted in the last three days (requires admin) + +**Options:** +- \`--json\` - Output as JSON (for piping to jq) +- \`--kind \` - Only items of this kind: script, flow, app, schedule, variable, resource or a trigger kind such as http_trigger +- \`--limit \` - Number of items to return (default 100, max 1000) +- \`--page \` - Page to return, starting at 1 + +**Subcommands:** + +- \`trash list\` - List trashed items, most recently deleted first + - \`--json\` - Output as JSON (for piping to jq) + - \`--kind \` - Only items of this kind: script, flow, app, schedule, variable, resource or a trigger kind such as http_trigger + - \`--limit \` - Number of items to return (default 100, max 1000) + - \`--page \` - Page to return, starting at 1 +- \`trash get \` - Show a trashed item and the data it was deleted with + - \`--json\` - Output as JSON (for piping to jq) +- \`trash restore \` - Put trashed items back at their paths + ### trigger trigger related commands diff --git a/cli/src/main.ts b/cli/src/main.ts index ccce20da30..fa894b6f2d 100755 --- a/cli/src/main.ts +++ b/cli/src/main.ts @@ -29,7 +29,7 @@ import lint from "./commands/lint/lint.ts"; import dev from "./commands/dev/dev.ts"; import { GlobalOptions } from "./types.ts"; import { OpenAPI } from "../gen/index.ts"; -import { getHeaders } from "./utils/utils.ts"; +import { apiErrorMessage, getHeaders } from "./utils/utils.ts"; import { detectAuthGatewayChallenge } from "./utils/http_guards.ts"; import { setShowDiffs } from "./core/conf.ts"; import { markRequestsAsCliClient } from "./core/client.ts"; @@ -48,6 +48,7 @@ import job from "./commands/job/job.ts"; import group from "./commands/group/group.ts"; import audit from "./commands/audit/audit.ts"; import token from "./commands/token/token.ts"; +import trash from "./commands/trash/trash.ts"; import generateMetadata from "./commands/generate-metadata/generate-metadata.ts"; import docs from "./commands/docs/docs.ts"; import config from "./commands/config/config.ts"; @@ -214,6 +215,7 @@ const command = new Command() .command("group", group) .command("audit", audit) .command("token", token) + .command("trash", trash) .command("generate-metadata", generateMetadata) .command("docs", docs) .command("config", config) @@ -321,14 +323,9 @@ async function main() { await command.parse(args); } catch (e) { - if (e && typeof e === "object" && "name" in e && e.name === "ApiError") { - const body = (e as any).body; - let bodyStr = typeof body === "object" && body !== null ? JSON.stringify(body) : String(body ?? ""); - // Strip backend source file references like (flows.rs:1400) or @scripts.rs:123:45 - bodyStr = bodyStr.replace(/\s*[@(]\w+\.rs:\d+[:\d]*\)?/g, ""); - log.error( - "Server failed. " + (e as any).statusText + ": " + bodyStr - ); + const apiError = apiErrorMessage(e); + if (apiError !== undefined) { + log.error("Server failed. " + apiError); } else if (e instanceof Error) { log.error(e.message); } else if (e !== undefined && e !== null) { diff --git a/cli/src/utils/utils.ts b/cli/src/utils/utils.ts index 489a877af7..0801c03911 100644 --- a/cli/src/utils/utils.ts +++ b/cli/src/utils/utils.ts @@ -353,6 +353,23 @@ export function formatTimestamp(ts: string): string { return new Date(ts).toISOString().replace("T", " ").substring(0, 19); } +/** + * ": " for an error thrown by the generated API client, + * undefined for anything else. Backend source references such as + * `(flows.rs:1400)` are stripped from the body. + */ +export function apiErrorMessage(e: unknown): string | undefined { + if (!(e && typeof e === "object" && "name" in e && e.name === "ApiError")) { + return undefined; + } + const { body, statusText } = e as { body?: unknown; statusText?: string }; + const bodyStr = + typeof body === "object" && body !== null + ? JSON.stringify(body) + : String(body ?? ""); + return statusText + ": " + bodyStr.replace(/\s*[@(]\w+\.rs:\d+[:\d]*\)?/g, ""); +} + /** * Validate that required arguments are present when no -d data was provided. * Fetches the schema from the API and checks required fields. diff --git a/cli/test/trash_commands.test.ts b/cli/test/trash_commands.test.ts new file mode 100644 index 0000000000..7b417dbc5e --- /dev/null +++ b/cli/test/trash_commands.test.ts @@ -0,0 +1,66 @@ +import { expect, test, describe } from "bun:test"; +import { withTestBackend } from "./test_backend.ts"; +import { setupWorkspaceProfile, ensureFolder } from "./new_commands_helpers.ts"; + +describe("trash command", () => { + test("lists, shows and restores a deleted variable", async () => { + await withTestBackend(async (backend, tempDir) => { + await setupWorkspaceProfile(backend); + await ensureFolder(backend, "test"); + const ws = backend.workspace; + const path = `f/test/trash_${Date.now()}`; + const api = (route: string, init: RequestInit = {}) => + backend.apiRequest!(`/api/w/${ws}/${route}`, { + headers: { "Content-Type": "application/json" }, + ...init, + }); + + let resp = await api("variables/create", { + method: "POST", + body: JSON.stringify({ path, value: "kept", is_secret: false, description: "" }), + }); + expect(resp.status).toBeLessThan(300); + await resp.text(); + resp = await api(`variables/delete/${path}`, { method: "DELETE" }); + expect(resp.status).toBeLessThan(300); + await resp.text(); + + const list = await backend.runCLICommand( + ["trash", "list", "--json", "--kind", "variable"], + tempDir + ); + expect(list.code).toBe(0); + const item = JSON.parse(list.stdout).find((i: any) => i.item_path === path); + expect(item).toBeDefined(); + expect(item.item_kind).toBe("variable"); + + const get = await backend.runCLICommand( + ["trash", "get", "--json", String(item.id)], + tempDir + ); + expect(get.code).toBe(0); + expect(JSON.parse(get.stdout).item_data.row.value).toBe("kept"); + + // A bogus second id: the first restore must still go through, and the + // failure must show in the exit code. + const restore = await backend.runCLICommand( + ["trash", "restore", String(item.id), "999999999"], + tempDir + ); + expect(restore.code).toBe(1); + expect(restore.stdout).toContain(`variable '${path}' restored`); + expect(restore.stderr).toContain("999999999"); + + resp = await api(`variables/get/${path}`); + expect(resp.status).toBe(200); + expect((await resp.json()).value).toBe("kept"); + + const after = await backend.runCLICommand( + ["trash", "list", "--json", "--kind", "variable"], + tempDir + ); + expect(after.code).toBe(0); + expect(JSON.parse(after.stdout).some((i: any) => i.item_path === path)).toBe(false); + }); + }); +}); diff --git a/frontend/src/lib/components/settings/Trashbin.svelte b/frontend/src/lib/components/settings/Trashbin.svelte index 3b9b4f5f29..ce45e7a7e2 100644 --- a/frontend/src/lib/components/settings/Trashbin.svelte +++ b/frontend/src/lib/components/settings/Trashbin.svelte @@ -7,7 +7,7 @@ import Row from '$lib/components/table/Row.svelte' import { workspaceStore } from '$lib/stores' import { sendUserToast } from '$lib/toast' - import { type TrashItem, TrashService } from '$lib/services/trashService' + import { type TrashItem, TrashService } from '$lib/gen' import { Trash2, RotateCcw, diff --git a/frontend/src/lib/services/trashService.ts b/frontend/src/lib/services/trashService.ts deleted file mode 100644 index c1bdac5a23..0000000000 --- a/frontend/src/lib/services/trashService.ts +++ /dev/null @@ -1,69 +0,0 @@ -import { OpenAPI } from '$lib/gen/core/OpenAPI' -import { request as __request } from '$lib/gen/core/request' - -export type TrashItem = { - id: number - workspace_id: string - item_kind: string - item_path: string - deleted_by: string - deleted_at: string - expires_at: string -} - -export class TrashService { - public static listTrash(data: { - workspace: string - itemKind?: string - page?: number - perPage?: number - }): Promise { - return __request(OpenAPI, { - method: 'GET', - url: '/w/{workspace}/trash/list', - path: { - workspace: data.workspace - }, - query: { - item_kind: data.itemKind, - page: data.page, - per_page: data.perPage - } - }) - } - - public static restoreTrashItem(data: { workspace: string; id: number }): Promise { - return __request(OpenAPI, { - method: 'POST', - url: '/w/{workspace}/trash/restore/{id}', - path: { - workspace: data.workspace, - id: data.id - } - }) - } - - public static permanentlyDeleteTrashItem(data: { - workspace: string - id: number - }): Promise { - return __request(OpenAPI, { - method: 'DELETE', - url: '/w/{workspace}/trash/delete/{id}', - path: { - workspace: data.workspace, - id: data.id - } - }) - } - - public static emptyTrash(data: { workspace: string }): Promise { - return __request(OpenAPI, { - method: 'POST', - url: '/w/{workspace}/trash/empty', - path: { - workspace: data.workspace - } - }) - } -} diff --git a/system_prompts/auto-generated/cli/cli-commands.md b/system_prompts/auto-generated/cli/cli-commands.md index 833c039ea8..b18638ee23 100644 --- a/system_prompts/auto-generated/cli/cli-commands.md +++ b/system_prompts/auto-generated/cli/cli-commands.md @@ -668,6 +668,27 @@ Manage API tokens - `--expiration ` - Token expiration (ISO 8601 timestamp) - `token delete ` - Delete a token by its prefix +### trash + +List, inspect and restore items deleted in the last three days (requires admin) + +**Options:** +- `--json` - Output as JSON (for piping to jq) +- `--kind ` - Only items of this kind: script, flow, app, schedule, variable, resource or a trigger kind such as http_trigger +- `--limit ` - Number of items to return (default 100, max 1000) +- `--page ` - Page to return, starting at 1 + +**Subcommands:** + +- `trash list` - List trashed items, most recently deleted first + - `--json` - Output as JSON (for piping to jq) + - `--kind ` - Only items of this kind: script, flow, app, schedule, variable, resource or a trigger kind such as http_trigger + - `--limit ` - Number of items to return (default 100, max 1000) + - `--page ` - Page to return, starting at 1 +- `trash get ` - Show a trashed item and the data it was deleted with + - `--json` - Output as JSON (for piping to jq) +- `trash restore ` - Put trashed items back at their paths + ### trigger trigger related commands diff --git a/system_prompts/auto-generated/prompts.ts b/system_prompts/auto-generated/prompts.ts index dc27234d60..216c72c128 100644 --- a/system_prompts/auto-generated/prompts.ts +++ b/system_prompts/auto-generated/prompts.ts @@ -3864,6 +3864,27 @@ Manage API tokens - \`--expiration \` - Token expiration (ISO 8601 timestamp) - \`token delete \` - Delete a token by its prefix +### trash + +List, inspect and restore items deleted in the last three days (requires admin) + +**Options:** +- \`--json\` - Output as JSON (for piping to jq) +- \`--kind \` - Only items of this kind: script, flow, app, schedule, variable, resource or a trigger kind such as http_trigger +- \`--limit \` - Number of items to return (default 100, max 1000) +- \`--page \` - Page to return, starting at 1 + +**Subcommands:** + +- \`trash list\` - List trashed items, most recently deleted first + - \`--json\` - Output as JSON (for piping to jq) + - \`--kind \` - Only items of this kind: script, flow, app, schedule, variable, resource or a trigger kind such as http_trigger + - \`--limit \` - Number of items to return (default 100, max 1000) + - \`--page \` - Page to return, starting at 1 +- \`trash get \` - Show a trashed item and the data it was deleted with + - \`--json\` - Output as JSON (for piping to jq) +- \`trash restore \` - Put trashed items back at their paths + ### trigger trigger related commands diff --git a/system_prompts/auto-generated/skills/cli-commands/SKILL.md b/system_prompts/auto-generated/skills/cli-commands/SKILL.md index e562938f48..3aafa0be85 100644 --- a/system_prompts/auto-generated/skills/cli-commands/SKILL.md +++ b/system_prompts/auto-generated/skills/cli-commands/SKILL.md @@ -673,6 +673,27 @@ Manage API tokens - `--expiration ` - Token expiration (ISO 8601 timestamp) - `token delete ` - Delete a token by its prefix +### trash + +List, inspect and restore items deleted in the last three days (requires admin) + +**Options:** +- `--json` - Output as JSON (for piping to jq) +- `--kind ` - Only items of this kind: script, flow, app, schedule, variable, resource or a trigger kind such as http_trigger +- `--limit ` - Number of items to return (default 100, max 1000) +- `--page ` - Page to return, starting at 1 + +**Subcommands:** + +- `trash list` - List trashed items, most recently deleted first + - `--json` - Output as JSON (for piping to jq) + - `--kind ` - Only items of this kind: script, flow, app, schedule, variable, resource or a trigger kind such as http_trigger + - `--limit ` - Number of items to return (default 100, max 1000) + - `--page ` - Page to return, starting at 1 +- `trash get ` - Show a trashed item and the data it was deleted with + - `--json` - Output as JSON (for piping to jq) +- `trash restore ` - Put trashed items back at their paths + ### trigger trigger related commands From d0cac0807f1b6f4fc76d6e5f7e27e03d30abec8d Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 14 Sep 2026 22:15:37 +0200 Subject: [PATCH 11/44] fix: set the enclosing span's trace context on exported log records (#11123) * chore: pin the EE ref that stamps trace context on exported log records Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01WfgEm5rNRDyw4WUYf9ToVz * chore: pin the EE ref with the sampling-decision test Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01WfgEm5rNRDyw4WUYf9ToVz * chore: update ee-repo-ref to 04a9f1efb4a52c79fcd20258b34780c86103d27f This commit updates the EE repository reference after PR #800 was merged in windmill-ee-private. Previous ee-repo-ref: d48361d66580618cb7a934d3c5f56a0c7e39ffaa New ee-repo-ref: 04a9f1efb4a52c79fcd20258b34780c86103d27f Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Fable 5.1 Co-authored-by: windmill-internal-app[bot] --- backend/ee-repo-ref.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 062211f925..a02a129b97 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -1ba6fe83451f0a1f8fafe04b7187087d51e0f769 +04a9f1efb4a52c79fcd20258b34780c86103d27f From e8c02c04cdb1a3f199f3f0a8b53a839a53d4a1d9 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 14 Sep 2026 22:32:45 +0200 Subject: [PATCH 12/44] feat: windmill-chat sdk for chat-mode flows in external frontends and raw apps (#11117) * feat: windmill-chat sdk for chat-mode flows in external frontends and raw apps Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_018aQiZNAU8g17kWkyTryS5J * fix: keep streamed answers until persisted, finish turns after history fallback Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_018aQiZNAU8g17kWkyTryS5J * feat: ai sdk transport and assistant-ui runtime for windmill-chat Co-Authored-By: Claude Fable 5.1 * fix: finish a turn from the flow result until its answer row lands, hash chat ids without crypto.subtle Co-Authored-By: Claude Fable 5.1 * fix: judge a turn answered by a persisted assistant row, wherever it was fetched Co-Authored-By: Claude Fable 5.1 * fix: attribute a turn's answer to its own jobs, keep a local turn when switching conversations Co-Authored-By: Claude Fable 5.1 * fix: mirror local history on every change, attribute failure-handler answers to the turn Co-Authored-By: Claude Fable 5.1 * fix: new chat per token string in the React hook, idle after destroy, no reorder on view Co-Authored-By: Claude Fable 5.1 * fix: recreate the hook's chat on any credential change, namespace local history per user Co-Authored-By: Claude Fable 5.1 * fix: send the latest inputs from the React hook Co-Authored-By: Claude Fable 5.1 --------- Co-authored-by: Claude Fable 5.1 --- .github/change-versions-mac.sh | 1 + .github/change-versions.sh | 3 + .github/workflows/npm_on_release.yml | 11 + .github/workflows/sdk-tests.yml | 19 + backend/windmill-api/openapi.yaml | 3 +- backend/windmill-api/src/apps.rs | 8 +- backend/windmill-api/src/lib.rs | 6 +- .../src/mcp/auto_generated_endpoints.rs | 16 +- chat-sdk/.gitignore | 2 + chat-sdk/README.md | 283 ++ chat-sdk/package-lock.json | 3451 +++++++++++++++++ chat-sdk/package.json | 108 + chat-sdk/src/ai-sdk.ts | 324 ++ chat-sdk/src/api.ts | 294 ++ chat-sdk/src/assistant-ui.ts | 124 + chat-sdk/src/chat.ts | 744 ++++ chat-sdk/src/config.ts | 81 + chat-sdk/src/follow.ts | 54 + chat-sdk/src/history.ts | 90 + chat-sdk/src/index.ts | 29 + chat-sdk/src/react.ts | 67 + chat-sdk/src/stream.ts | 65 + chat-sdk/src/types.ts | 123 + chat-sdk/src/utils.ts | 134 + chat-sdk/test/ai-sdk-chat.test.ts | 62 + chat-sdk/test/ai-sdk.test.ts | 178 + chat-sdk/test/assistant-ui.test.ts | 38 + chat-sdk/test/chat.test.ts | 671 ++++ chat-sdk/test/config.test.ts | 49 + chat-sdk/test/react.test.tsx | 80 + chat-sdk/test/stream.test.ts | 61 + chat-sdk/test/support.ts | 101 + chat-sdk/tsconfig.build.json | 12 + chat-sdk/tsconfig.json | 15 + chat-sdk/tsdown.config.ts | 9 + cli/src/guidance/skills.gen.ts | 14 + .../src/lib/components/raw_apps/sdkScopes.ts | 12 +- frontend/src/lib/mcpEndpointTools.ts | 16 +- system_prompts/auto-generated/prompts.ts | 14 + .../auto-generated/skills/raw-app/SKILL.md | 14 + system_prompts/base/raw-app.md | 14 + 41 files changed, 7384 insertions(+), 16 deletions(-) create mode 100644 chat-sdk/.gitignore create mode 100644 chat-sdk/README.md create mode 100644 chat-sdk/package-lock.json create mode 100644 chat-sdk/package.json create mode 100644 chat-sdk/src/ai-sdk.ts create mode 100644 chat-sdk/src/api.ts create mode 100644 chat-sdk/src/assistant-ui.ts create mode 100644 chat-sdk/src/chat.ts create mode 100644 chat-sdk/src/config.ts create mode 100644 chat-sdk/src/follow.ts create mode 100644 chat-sdk/src/history.ts create mode 100644 chat-sdk/src/index.ts create mode 100644 chat-sdk/src/react.ts create mode 100644 chat-sdk/src/stream.ts create mode 100644 chat-sdk/src/types.ts create mode 100644 chat-sdk/src/utils.ts create mode 100644 chat-sdk/test/ai-sdk-chat.test.ts create mode 100644 chat-sdk/test/ai-sdk.test.ts create mode 100644 chat-sdk/test/assistant-ui.test.ts create mode 100644 chat-sdk/test/chat.test.ts create mode 100644 chat-sdk/test/config.test.ts create mode 100644 chat-sdk/test/react.test.tsx create mode 100644 chat-sdk/test/stream.test.ts create mode 100644 chat-sdk/test/support.ts create mode 100644 chat-sdk/tsconfig.build.json create mode 100644 chat-sdk/tsconfig.json create mode 100644 chat-sdk/tsdown.config.ts diff --git a/.github/change-versions-mac.sh b/.github/change-versions-mac.sh index 2f23b21eb8..72bb2b9d8c 100755 --- a/.github/change-versions-mac.sh +++ b/.github/change-versions-mac.sh @@ -12,6 +12,7 @@ sed -i '' -e "/^export const VERSION =/s/= .*/= \"v$VERSION\";/" ${root_dirpath} sed -i '' -e "/version: /s/: .*/: $VERSION/" ${root_dirpath}/backend/windmill-api/openapi.yaml sed -i '' -e "/version: /s/: .*/: $VERSION/" ${root_dirpath}/openflow.openapi.yaml sed -i '' -e "/\"version\": /s/: .*,/: \"$VERSION\",/" ${root_dirpath}/typescript-client/package.json +sed -i '' -e "/\"version\": /s/: .*,/: \"$VERSION\",/" ${root_dirpath}/chat-sdk/package.json sed -i '' -e "/\"version\": /s/: .*,/: \"$VERSION\",/" ${root_dirpath}/frontend/package.json sed -i '' -e "/^version =/s/= .*/= \"$VERSION\"/" ${root_dirpath}/python-client/wmill/pyproject.toml sed -i '' -e "/^windmill-api =/s/= .*/= \"\\^$VERSION\"/" ${root_dirpath}/python-client/wmill/pyproject.toml diff --git a/.github/change-versions.sh b/.github/change-versions.sh index ffb73f6180..1772ddbc89 100755 --- a/.github/change-versions.sh +++ b/.github/change-versions.sh @@ -13,6 +13,7 @@ sed -i -e "/version: /s/: .*/: $VERSION/" ${root_dirpath}/backend/windmill-api/o sed -i -e "/version: /s/: .*/: $VERSION/" ${root_dirpath}/openflow.openapi.yaml sed -i -e "/\"version\": /s/: .*,/: \"$VERSION\",/" ${root_dirpath}/typescript-client/package.json sed -i -e "/\"version\": /s/: .*,/: \"$VERSION\",/" ${root_dirpath}/typescript-client/jsr.json +sed -i -e "/\"version\": /s/: .*,/: \"$VERSION\",/" ${root_dirpath}/chat-sdk/package.json sed -i -e "/\"version\": /s/: .*,/: \"$VERSION\",/" ${root_dirpath}/frontend/package.json sed -i -e "/\"version\": /s/: .*,/: \"$VERSION\",/" ${root_dirpath}/windmill-yaml-validator/package.json sed -i -e "/^version =/s/= .*/= \"$VERSION\"/" ${root_dirpath}/python-client/wmill/pyproject.toml @@ -33,3 +34,5 @@ cd ${root_dirpath}/frontend && npm i --package-lock-only --ignore-scripts # The CLI installs this package on every `bun install`, which would otherwise rewrite the # lockfile's version and leave a dirty tree. cd ${root_dirpath}/windmill-yaml-validator && npm i --package-lock-only --ignore-scripts + +cd ${root_dirpath}/chat-sdk && npm i --package-lock-only --ignore-scripts diff --git a/.github/workflows/npm_on_release.yml b/.github/workflows/npm_on_release.yml index a41bd80854..7d30265c74 100644 --- a/.github/workflows/npm_on_release.yml +++ b/.github/workflows/npm_on_release.yml @@ -17,6 +17,17 @@ jobs: - run: cd typescript-client && ./publish.sh --access public && cd .. env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + publish_chat_sdk: + runs-on: ubicloud-standard-8 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v3 + with: + node-version: "20.x" + registry-url: "https://registry.npmjs.org" + - run: cd chat-sdk && npm ci && npm run build && npm publish --access public + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} publish_cli: runs-on: ubicloud-standard-8 steps: diff --git a/.github/workflows/sdk-tests.yml b/.github/workflows/sdk-tests.yml index efcbcd6ec1..6f7d79c6af 100644 --- a/.github/workflows/sdk-tests.yml +++ b/.github/workflows/sdk-tests.yml @@ -32,6 +32,25 @@ jobs: working-directory: ./typescript-client run: bun test --timeout 120000 tests/ + chat-sdk: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Bun + uses: oven-sh/setup-bun@v2 + with: + bun-version: latest + + - uses: actions/setup-node@v4 + with: + node-version: "20.x" + + - name: Run tests + working-directory: ./chat-sdk + run: npm ci && npm run check && bun test + python-client: runs-on: ubuntu-latest steps: diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 21c059bc1a..aee198e92a 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -34334,7 +34334,8 @@ components: carrying their own identity restricted to these scopes, handed to the app bundle so `windmill-client` calls run as the viewer. Must be a subset of the server's curated allowlist (jobs:run, jobs:read, - users:read, resources:read, variables:read). + users:read, resources:read, variables:read, flow_conversations:read, + flow_conversations:write). ListableApp: type: object diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index daa7fee59b..cc4c5079f9 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -1440,12 +1440,14 @@ const APP_EMBED_TOKEN_VALIDITY_HOURS: i64 = 12; /// Scopes an app author may declare in `Policy::frontend_sdk_scopes`. No `apps:*` /// scope, so the token cannot reach the mint endpoints and renew itself; the /// `raw_app_sdk` sentinel narrows the rest (see `scopes.rs`). -pub const FRONTEND_SDK_ALLOWED_SCOPES: [&str; 5] = [ +pub const FRONTEND_SDK_ALLOWED_SCOPES: [&str; 7] = [ "jobs:run", "jobs:read", "users:read", "resources:read", "variables:read", + "flow_conversations:read", + "flow_conversations:write", ]; /// Reject a policy declaring frontend SDK scopes outside the curated list. @@ -6014,6 +6016,10 @@ mod embed_token_tests { // the author declared it and the viewer consented. ("/api/w/test/resources/get_value/u/admin/r", "GET"), ("/api/w/test/variables/get_value/u/admin/v", "GET"), + // A chat UI's history for a chat-mode flow; RLS keeps it to the viewer's own. + ("/api/w/test/flow_conversations/list", "GET"), + ("/api/w/test/flow_conversations/some-uuid/messages", "GET"), + ("/api/w/test/flow_conversations/delete/some-uuid", "DELETE"), ]; for (path, method) in allowed { assert!( diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index fc2c773703..bd6fda015b 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -659,9 +659,13 @@ pub async fn run_server( "/workspace_dependencies", workspace_dependencies::workspaced_service(), ) + // CORS so a chat UI on another origin (an external site, or + // a sandboxed raw app with its frontend SDK token) can read + // its conversation history. Bearer-only, like variables. .nest( "/flow_conversations", - windmill_api_flow_conversations::workspaced_service(), + windmill_api_flow_conversations::workspaced_service() + .layer(cors.clone()), ) // CORS so an opaque-origin app iframe (WIN-2006 embed, // no separate domain) can read folders/listnames with a diff --git a/backend/windmill-api/src/mcp/auto_generated_endpoints.rs b/backend/windmill-api/src/mcp/auto_generated_endpoints.rs index ea673610e6..ffefaba2dd 100644 --- a/backend/windmill-api/src/mcp/auto_generated_endpoints.rs +++ b/backend/windmill-api/src/mcp/auto_generated_endpoints.rs @@ -1268,10 +1268,12 @@ is, a different one moves it there and archives the old path"), "description": "Who may open the app, and who its runnables execute as. Optional, and what omitting it means depends on the operation: creating an app defaults it to `publisher` (runs on behalf of the app's publisher and requires an authenticated viewer), while updating one keeps the mode the app is already deployed under. Neither `anonymous`, which makes the app publicly executable, nor `guest`, which opens it to anyone the identity provider authenticates, is ever assumed. A guest is only admitted where the workspace also has `guest_access_enabled`, which is checked when the session is minted and again on every guest request. Possible values: viewer, publisher, guest, anonymous" }, "on_behalf_of": { - "type": "string" + "type": "string", + "description": "The user or group the app runs as in anonymous or publisher mode (e.g. 'u/admin' or 'g/mygroup'). The authority for the app's identity." }, "on_behalf_of_email": { - "type": "string" + "type": "string", + "description": "Address of `on_behalf_of`, written through from it on every save and returned as stored. Optional; when absent it is derived from `on_behalf_of`. Sending it is optional too; it must name the same account as `on_behalf_of`, and a pair that disagrees is rejected." }, "sandbox": { "type": "boolean", @@ -1282,7 +1284,7 @@ is, a different one moves it there and archives the old path"), "items": { "type": "string" }, - "description": "Raw apps: author-declared scopes for the frontend SDK token. Takes effect only when `sandbox` is also true — an unsandboxed bundle runs with the viewer's own session, so no token is advertised or minted for it and this list stays inert. On a sandboxed app a non-empty list lets viewers mint (after consenting) a short-lived token carrying their own identity restricted to these scopes, handed to the app bundle so `windmill-client` calls run as the viewer. Must be a subset of the server's curated allowlist (jobs:run, jobs:read, users:read, resources:read, variables:read).\n" + "description": "Raw apps: author-declared scopes for the frontend SDK token. Takes effect only when `sandbox` is also true — an unsandboxed bundle runs with the viewer's own session, so no token is advertised or minted for it and this list stays inert. On a sandboxed app a non-empty list lets viewers mint (after consenting) a short-lived token carrying their own identity restricted to these scopes, handed to the app bundle so `windmill-client` calls run as the viewer. Must be a subset of the server's curated allowlist (jobs:run, jobs:read, users:read, resources:read, variables:read, flow_conversations:read, flow_conversations:write).\n" } } } @@ -1383,10 +1385,12 @@ is, a different one moves it there and archives the old path"), "description": "Who may open the app, and who its runnables execute as. Optional, and what omitting it means depends on the operation: creating an app defaults it to `publisher` (runs on behalf of the app's publisher and requires an authenticated viewer), while updating one keeps the mode the app is already deployed under. Neither `anonymous`, which makes the app publicly executable, nor `guest`, which opens it to anyone the identity provider authenticates, is ever assumed. A guest is only admitted where the workspace also has `guest_access_enabled`, which is checked when the session is minted and again on every guest request. Possible values: viewer, publisher, guest, anonymous" }, "on_behalf_of": { - "type": "string" + "type": "string", + "description": "The user or group the app runs as in anonymous or publisher mode (e.g. 'u/admin' or 'g/mygroup'). The authority for the app's identity." }, "on_behalf_of_email": { - "type": "string" + "type": "string", + "description": "Address of `on_behalf_of`, written through from it on every save and returned as stored. Optional; when absent it is derived from `on_behalf_of`. Sending it is optional too; it must name the same account as `on_behalf_of`, and a pair that disagrees is rejected." }, "sandbox": { "type": "boolean", @@ -1397,7 +1401,7 @@ is, a different one moves it there and archives the old path"), "items": { "type": "string" }, - "description": "Raw apps: author-declared scopes for the frontend SDK token. Takes effect only when `sandbox` is also true — an unsandboxed bundle runs with the viewer's own session, so no token is advertised or minted for it and this list stays inert. On a sandboxed app a non-empty list lets viewers mint (after consenting) a short-lived token carrying their own identity restricted to these scopes, handed to the app bundle so `windmill-client` calls run as the viewer. Must be a subset of the server's curated allowlist (jobs:run, jobs:read, users:read, resources:read, variables:read).\n" + "description": "Raw apps: author-declared scopes for the frontend SDK token. Takes effect only when `sandbox` is also true — an unsandboxed bundle runs with the viewer's own session, so no token is advertised or minted for it and this list stays inert. On a sandboxed app a non-empty list lets viewers mint (after consenting) a short-lived token carrying their own identity restricted to these scopes, handed to the app bundle so `windmill-client` calls run as the viewer. Must be a subset of the server's curated allowlist (jobs:run, jobs:read, users:read, resources:read, variables:read, flow_conversations:read, flow_conversations:write).\n" } } }, diff --git a/chat-sdk/.gitignore b/chat-sdk/.gitignore new file mode 100644 index 0000000000..1eae0cf670 --- /dev/null +++ b/chat-sdk/.gitignore @@ -0,0 +1,2 @@ +dist/ +node_modules/ diff --git a/chat-sdk/README.md b/chat-sdk/README.md new file mode 100644 index 0000000000..98874a719f --- /dev/null +++ b/chat-sdk/README.md @@ -0,0 +1,283 @@ +# windmill-chat + +Build a chat interface on a Windmill flow deployed in **chat mode**, from any frontend +or from a Windmill raw app. The library is headless: it runs the flow, follows the +answer as it streams, keeps the conversation history, and hands you state to render. + +``` +npm install windmill-chat +``` + +No runtime dependencies. Optional peers: `react` for `windmill-chat/react`, `ai` for +`windmill-chat/ai-sdk`, `@assistant-ui/react` for `windmill-chat/assistant-ui`. + +| You build the UI with | Import | You get | +|---|---|---| +| Vercel AI SDK `useChat`, AI Elements | `windmill-chat/ai-sdk` | a `ChatTransport`: `useChat({ transport })`, nothing else changes | +| assistant-ui | `windmill-chat/assistant-ui` | a runtime for `AssistantRuntimeProvider`, threads included | +| Your own components | `windmill-chat/react` or `windmill-chat` | a hook / a store with messages, status and actions | + +## The flow + +Any deployed flow with **Chat mode** enabled in its settings works. Windmill passes the +message as the `user_message` input and threads the conversation through `memory_id`, +so an AI agent step remembers earlier turns. The answer is: + +- what the last step streams, when it is an AI agent step; +- otherwise the flow's result: its `windmill_chat_answer` field when it has one, a + string as is, anything else as JSON. + +## Vercel AI SDK (`useChat`, AI Elements) + +```tsx +import { useChat } from '@ai-sdk/react' +import { createWindmillChatTransport } from 'windmill-chat/ai-sdk' + +const transport = createWindmillChatTransport({ + baseUrl: 'https://app.windmill.dev', + workspace: 'acme', + flowPath: 'f/support/assistant', + token: () => fetch('/api/windmill-token').then((r) => r.text()) +}) + +export function Support() { + const { messages, status, sendMessage, stop } = useChat({ id: conversationId, transport }) + // render `messages[i].parts`: text, reasoning and dynamic-tool parts, as with any AI SDK backend +} +``` + +The chat `id` is the conversation: reuse it to continue one, and pass a UUID when you +also read server history, so it matches what `flow_conversations` stores (any other id +maps to a fixed UUID). `sendMessage(msg, { body })` sends extra flow inputs. Tool calls +arrive as `dynamic-tool` parts (`input-available → output-available | output-error`), +which AI Elements' `` renders as is. A failed flow surfaces as `error`. +`regenerate()` runs the flow again with the same message: a new turn on the server, +not a replacement of the previous answer. + +The transport also carries the history helpers: `transport.loadMessages(id)` returns +`UIMessage`s for `useChat({ messages })` or `setMessages`, `transport.listConversations()` +and `transport.deleteConversation(id)`. Attachments are not supported: `sendMessage` with +`files` is refused with an explanatory error. + +## assistant-ui + +```tsx +import { AssistantRuntimeProvider } from '@assistant-ui/react' +import { useWindmillRuntime } from 'windmill-chat/assistant-ui' + +export function Support() { + const runtime = useWindmillRuntime({ baseUrl, workspace, flowPath, token }) + return ( + + {/* your assistant-ui components, thread list included */} + + ) +} +``` + +Conversations are threads: `ThreadListPrimitive` switches, creates and deletes them. +Tool calls render through your `tools` components (`MessagePrimitive.Parts`), reasoning +through `Reasoning`. It takes the same options as `useWindmillChat` below. + +## React + +```tsx +import { useWindmillChat } from 'windmill-chat/react' + +export function Support() { + const chat = useWindmillChat({ + baseUrl: 'https://app.windmill.dev', + workspace: 'acme', + flowPath: 'f/support/assistant', + token: () => fetch('/api/windmill-token').then((r) => r.text()) + }) + const [draft, setDraft] = useState('') + + return ( +
    + {chat.messages.map((m) => ( +

    + {m.content} +

    + ))} +
    { + e.preventDefault() + chat.sendMessage(draft) + setDraft('') + }} + > + setDraft(e.target.value)} /> + + {chat.status === 'streaming' && ( + + )} +
    +
    + ) +} +``` + +The hook returns the [state](#state) plus the chat's methods. It recreates the chat +(fresh state, old one destroyed) when `flowPath`, `baseUrl`, `workspace`, `history`, +`storageKey` or the credential change: a different token string, or a switch between +no token, a string and a function. A token function is called through a ref, so +passing a new closure on every render is fine and never resets the chat; when users +sign in and out behind a token function, change `storageKey` (their id) so local +history and state start over with them. + +## Raw apps + +Inside a Windmill raw app nothing needs configuring: the chat runs as the viewer, +against the Windmill the app is served from. + +```tsx +const chat = useWindmillChat({ flowPath: 'f/support/assistant' }) +``` + +- **Unsandboxed app** (the default): the viewer's session is used. Viewers need + permission to run the flow. +- **Sandboxed app**: declare `jobs:run` in the app's frontend SDK scopes, and + `flow_conversations:write` for server-side history. The viewer consents once and + the app receives a token restricted to those scopes. +- **`wmill app dev`**: there is no viewer session on the dev server, so pass + `baseUrl`, `workspace` and `token` explicitly during development. + +## Any framework + +`createChat` returns a store: `subscribe` calls the listener immediately and on every +change, and returns the unsubscribe function. That is the Svelte store contract, so +`$chat` works as is; other frameworks wrap it in a few lines. + +```ts +import { createChat } from 'windmill-chat' + +const chat = createChat({ baseUrl, workspace, flowPath, token }) +chat.subscribe((state) => render(state)) +await chat.sendMessage('Hello') +``` + +```svelte + + +{#each $chat.messages as m (m.id)} +

    {m.content}

    +{/each} + +``` + +## Options + +| Option | | +|---|---| +| `flowPath` | Path of the deployed flow, e.g. `f/support/assistant`. Required. | +| `baseUrl` | The Windmill origin. Detected inside a raw app. | +| `workspace` | Detected inside a raw app. | +| `token` | A token, or a function returning one (called before every request, so it can fetch a short-lived token from your backend). Omit it inside a raw app. | +| `history` | `'server'`, `'local'` or `'none'`, see [History](#history). Defaults to `'server'` with a viewer session and `'local'` with an explicit `token`. | +| `inputs` | Extra flow inputs sent with every message. `sendMessage(text, { inputs })` adds per-message ones. | +| `storageKey` | Namespace for `local` history, e.g. the signed-in user's id. Local history is per browser and per flow; without it, users sharing a browser share it. | +| `fetch`, `storage` | Replacements for the globals, for tests and unusual runtimes. | +| `pageSize` | Messages and conversations per page of server history. Default 50. | +| `onFinish`, `onError` | Called when a turn has its answer, or could not run at all. | + +## State + +```ts +interface ChatState { + conversationId: string | undefined + messages: ChatMessage[] + status: 'idle' | 'submitted' | 'streaming' | 'error' + error: Error | undefined + conversations: Conversation[] + history: 'server' | 'local' | 'none' + loadingMessages: boolean + hasMoreMessages: boolean +} + +interface ChatMessage { + id: string + role: 'user' | 'assistant' | 'tool' | 'system' + content: string + reasoning?: string // the model's reasoning summary, when streamed + tool?: { callId?: string; name: string; arguments?: string; result?: string; status: 'running' | 'success' | 'error' } + success: boolean // false for a failed flow or tool + pending: boolean // still streaming, or not yet confirmed by the server + createdAt: string + jobId?: string + stepName?: string + serverId?: string // the persisted row; `id` itself never changes, so list keys are stable +} +``` + +A turn goes `submitted` (the flow is queued) → `streaming` (the answer is arriving) → +`idle`. Tool calls appear as `tool` messages whose `status` moves from `running` to +`success` or `error`. A flow that fails still completes the turn: its error is the +answer, an `assistant` message with `success: false`. `status: 'error'` (with `error` +set) means the turn could not run or be followed at all, such as a refused request. + +Methods: `sendMessage(text, { inputs? })`, `stop()`, `newConversation()`, +`selectConversation(id)`, `loadConversations({ page?, perPage? })`, +`deleteConversation(id)`, `loadOlderMessages()`, `destroy()`. Switching conversations +stops following the current answer; the flow keeps running and, with server history, +its answer is there when you come back. + +## History + +Windmill stores every conversation of a chat-mode flow, and each Windmill user sees +only their own. `history: 'server'` reads that store: `loadConversations()` lists +them, `selectConversation(id)` loads one, `loadOlderMessages()` pages back. Every +message rendered from the server carries its `jobId` and `stepName`. + +That store is keyed by the **Windmill user**, so it fits a viewer session or a token +issued per user. With one token shared by every visitor of a site, all visitors would +see each other's conversations. For that setup use `history: 'local'` (the default +with an explicit `token`): the conversation list and messages stay in the browser's +`localStorage`, per Windmill instance, workspace and flow. `'none'` keeps nothing +beyond the page. + +When the default `'server'` mode turns out unreadable (a token or sandboxed app +without `flow_conversations` scopes), the chat switches itself to `'local'` and +`state.history` says so. Passing `history` explicitly disables that fallback. + +## Tokens + +Anything a browser holds can be read by its user, so give a chat token exactly what +the chat needs: + +| Setup | Scopes | +|---|---| +| Public site, one token for everyone | `jobs:run:flows:f/support/assistant`, and `history: 'local'`. The token can run that one flow and follow its jobs, nothing else. | +| Per-user tokens minted by your backend | The above plus `flow_conversations:write`, with `history: 'server'` (an explicit token defaults to local history). Return them from an endpoint and pass `token: () => fetch(...)`. | +| A Windmill user in the browser (raw app, embedded Windmill) | No token: the session is used. | + +The token's user must be allowed to run the flow. `stop()` closes the stream in any +case; cancelling the run on the server as well needs `jobs:write`, which also lets the +token read every job its user can see, so leave it out unless that matters. + +Anyone holding the token can run the flow with inputs of their choosing, so a flow +exposed this way should treat `user_message` and the other inputs as untrusted. + +## Lower level + +`WindmillChatApi` wraps the endpoints (`runFlow`, `streamJob`, `listConversations`, +`listMessages`, `deleteConversation`, `cancelJob`), `followJob` follows a run to +completion across the server's stream timeouts, `parseStreamEvents` decodes the AI +agent stream, `extractChatAnswer` turns a flow result into the text a chat shows, and +`conversationIdFor` maps any chat id to its conversation UUID. They are exported for +custom integrations. + +## For AI coding agents + +When asked to add a chat over a Windmill flow: the flow must be deployed with chat mode +on. Pick the entry point from the table at the top (`useChat` → `windmill-chat/ai-sdk`, +assistant-ui → `windmill-chat/assistant-ui`, otherwise `windmill-chat/react`). Inside a +Windmill raw app pass only `flowPath`. Elsewhere pass `baseUrl`, `workspace` and a +`token`; for a public page use a token scoped to `jobs:run:flows:` and leave +`history` at its default. Render `role`, `content`, `pending`, `success` and +`tool.status`; never build the SSE handling yourself. diff --git a/chat-sdk/package-lock.json b/chat-sdk/package-lock.json new file mode 100644 index 0000000000..7d4ff28c21 --- /dev/null +++ b/chat-sdk/package-lock.json @@ -0,0 +1,3451 @@ +{ + "name": "windmill-chat", + "version": "1.810.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "windmill-chat", + "version": "1.810.0", + "license": "Apache-2.0", + "devDependencies": { + "@ai-sdk/react": "^4.0.102", + "@assistant-ui/react": "^0.15.19", + "@happy-dom/global-registrator": "^20.14.5", + "@types/bun": "^1.3.0", + "@types/react": "^19.0.0", + "@types/react-dom": "^19.3.0", + "ai": "^7.0.99", + "react": "^19.0.0", + "react-dom": "^19.3.0", + "tsdown": "^0.12.9", + "typescript": "^5.4.5" + }, + "peerDependencies": { + "@assistant-ui/react": ">=0.15", + "ai": ">=5", + "react": ">=18" + }, + "peerDependenciesMeta": { + "@assistant-ui/react": { + "optional": true + }, + "ai": { + "optional": true + }, + "react": { + "optional": true + } + } + }, + "node_modules/@ai-sdk/gateway": { + "version": "4.0.80", + "resolved": "https://registry.npmjs.org/@ai-sdk/gateway/-/gateway-4.0.80.tgz", + "integrity": "sha512-6t+07o8lSthpKf64Xb1qHWR2bWvJ3Fd2oFvS9fQc45p31bi2OUqan246e/ojAmZpWCiMPjKyQ4TBr4MYytnTiQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "4.0.14", + "@ai-sdk/provider-utils": "5.0.40", + "@vercel/oidc": "3.2.0" + }, + "engines": { + "node": ">=22" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/mcp": { + "version": "2.0.49", + "resolved": "https://registry.npmjs.org/@ai-sdk/mcp/-/mcp-2.0.49.tgz", + "integrity": "sha512-dD8//65te2C5B4UzPNHGLR7CStR5IvY7o7kXz0YYtPQs+aCdXc7Jb3fuvZcovShfP1KGvPIMTfXhp6Uz+UghJQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "4.0.14", + "@ai-sdk/provider-utils": "5.0.40", + "cross-spawn": "^7.0.6", + "pkce-challenge": "^5.0.1" + }, + "engines": { + "node": ">=22" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/provider": { + "version": "4.0.14", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-4.0.14.tgz", + "integrity": "sha512-yukP2tbcQQErG5gLCMBGvpvb/rM3D3KlTKG6eKKOdNHLHqtNNDEeBxdYrY/JL+O76B2ig5dXY19H/f1HFSvRiQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=22" + } + }, + "node_modules/@ai-sdk/provider-utils": { + "version": "5.0.40", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-5.0.40.tgz", + "integrity": "sha512-zsXPwSAQ9mRJ2hvyITaLOYUyuGrBmzFhJXOg4mllGmla1PfNxZcm4GwqMiV2xQaDgcgBMdUGxXkp9xekZZNIkg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "4.0.14", + "@standard-schema/spec": "^1.1.0", + "@workflow/serde": "4.1.0", + "eventsource-parser": "^3.0.8", + "undici": "^7.29.0" + }, + "engines": { + "node": ">=22" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/react": { + "version": "4.0.102", + "resolved": "https://registry.npmjs.org/@ai-sdk/react/-/react-4.0.102.tgz", + "integrity": "sha512-KOTRsaVUr6QeisktVqm57KfBMqBpxK9f2ay+defGnYsfvXpFE277zg107Q27LSpiTsDpmk2GHMqNWdU9bKaJSw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/mcp": "2.0.49", + "@ai-sdk/provider": "4.0.14", + "@ai-sdk/provider-utils": "5.0.40", + "ai": "7.0.99", + "swr": "^2.4.1", + "throttleit": "2.1.0" + }, + "engines": { + "node": ">=22" + }, + "peerDependencies": { + "react": "^18 || ~19.0.1 || ~19.1.2 || ^19.2.1" + } + }, + "node_modules/@assistant-ui/core": { + "version": "0.3.18", + "resolved": "https://registry.npmjs.org/@assistant-ui/core/-/core-0.3.18.tgz", + "integrity": "sha512-WR5/uqZuI6FNWogIe5EKmP7gdLhQhX38QqaAMAfH9DEPVzw40ON4sZC9vD47rpBj6yrXhHIsbt6jAGUKH4mASQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "assistant-stream": "^0.3.42", + "nanoid": "^6.0.1" + }, + "peerDependencies": { + "@assistant-ui/store": "^0.3.13", + "@assistant-ui/tap": "^0.9.17", + "@types/react": "*", + "assistant-cloud": "^0.2.0", + "react": "^18 || ^19" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "assistant-cloud": { + "optional": true + }, + "react": { + "optional": true + } + } + }, + "node_modules/@assistant-ui/react": { + "version": "0.15.19", + "resolved": "https://registry.npmjs.org/@assistant-ui/react/-/react-0.15.19.tgz", + "integrity": "sha512-+mEXA/ibBSoodj3LFUzRAdYZ+HvlrIg8uexPVjcY3ssksG1zHx8E9TpuwCDxg2h3XrYU8c/UZj/uFlI5tTWMoA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@assistant-ui/core": "^0.3.18", + "@assistant-ui/store": "^0.3.13", + "@assistant-ui/tap": "^0.9.17", + "assistant-cloud": "^0.2.0", + "assistant-stream": "^0.3.42", + "radix-ui": "^1.6.7", + "react-textarea-autosize": "^8.5.9", + "safe-content-frame": "^0.0.30", + "zod": "^4.5.4", + "zustand": "^5.0.15" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^18 || ^19", + "react-dom": "^18 || ^19" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@assistant-ui/store": { + "version": "0.3.13", + "resolved": "https://registry.npmjs.org/@assistant-ui/store/-/store-0.3.13.tgz", + "integrity": "sha512-4u5YAMfjgr+jpJUWZ5f1uqXQuxvIiR8SRg1WWlFbIknk5S1Wli4Jp/nTQsOeXsIw0d7Zdb6FKiQw31gqs5z/Ew==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@assistant-ui/tap": "^0.9.17", + "@types/react": "*", + "react": "^18 || ^19" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "react": { + "optional": true + } + } + }, + "node_modules/@assistant-ui/tap": { + "version": "0.9.17", + "resolved": "https://registry.npmjs.org/@assistant-ui/tap/-/tap-0.9.17.tgz", + "integrity": "sha512-z53TiHiM3ai8XQ5B60Lg8IKCWM0XRf1Sz1jTQtqiuAATzVd6zeFedSHpxq4i5P0zi5uYVdN/dkGEpVTOyPc3ow==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^18 || ^19" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "react": { + "optional": true + } + } + }, + "node_modules/@babel/generator": { + "version": "7.29.8", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.29.8", + "@babel/types": "^7.29.8", + "@jridgewell/gen-mapping": "^0.3.12", + "@jridgewell/trace-mapping": "^0.3.28", + "jsesc": "^3.0.2" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-string-parser": { + "version": "7.29.7", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.29.7", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/parser": { + "version": "7.29.8", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.29.8" + }, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/runtime": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.29.7.tgz", + "integrity": "sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/types": { + "version": "7.29.8", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-string-parser": "^7.29.7", + "@babel/helper-validator-identifier": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@floating-ui/core": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/@floating-ui/core/-/core-1.8.0.tgz", + "integrity": "sha512-0CIZ5itps/8x7BG8dEIhs53BvCUH2PCoogtakwRTut+Arm58sJooJ0AuZhLw2HJYIR5cMLNPBSS728sPho2khQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@floating-ui/utils": "^0.2.12" + } + }, + "node_modules/@floating-ui/dom": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/@floating-ui/dom/-/dom-1.8.0.tgz", + "integrity": "sha512-yXSrzeHZBTZadLOlfyhCkJHNeLJnHRnRInwdZ40L7ZiaAtrBwoYlsDrX3v5zB1Utk7CLfzcOVnVVWoXEky7Ceg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@floating-ui/core": "^1.8.0", + "@floating-ui/utils": "^0.2.12" + } + }, + "node_modules/@floating-ui/react-dom": { + "version": "2.1.9", + "resolved": "https://registry.npmjs.org/@floating-ui/react-dom/-/react-dom-2.1.9.tgz", + "integrity": "sha512-JDjEFGCpImxDCA7JJKviA0M9+RtmJdj0m/NVU5IMgBK+AmZouAQQ7/+2GLH0GXXY0YMw9oXPB8hKdbPYg5QLYg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@floating-ui/dom": "^1.8.0" + }, + "peerDependencies": { + "react": ">=16.8.0", + "react-dom": ">=16.8.0" + } + }, + "node_modules/@floating-ui/utils": { + "version": "0.2.12", + "resolved": "https://registry.npmjs.org/@floating-ui/utils/-/utils-0.2.12.tgz", + "integrity": "sha512-HpCo8tmWzLVad5s2d19EhAz5zqrrQ6s69qd6moPMQvkOuSwDT1YgRfWSVuc4ennqrgv3OHppiOGMQ7oC13yIww==", + "dev": true, + "license": "MIT" + }, + "node_modules/@happy-dom/global-registrator": { + "version": "20.14.5", + "resolved": "https://registry.npmjs.org/@happy-dom/global-registrator/-/global-registrator-20.14.5.tgz", + "integrity": "sha512-B05ID9DhSwLs6mlm1fzlkAtTIvB3duCvjJjfr19LBrlTK7VZtRjDqoTRIVv13GuYuNdhByu8LSZgThwV3Rkj7g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": ">=20.0.0", + "happy-dom": "^20.14.5" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.13", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.0", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.6.0", + "dev": true, + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.31", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@oxc-project/types": { + "version": "0.149.0", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/oxc-project" + } + }, + "node_modules/@quansync/fs": { + "version": "1.0.0", + "dev": true, + "license": "MIT", + "dependencies": { + "quansync": "^1.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sxzz" + } + }, + "node_modules/@radix-ui/number": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@radix-ui/number/-/number-1.1.3.tgz", + "integrity": "sha512-Road2bidD0uu/1BGDOWNdPI06g0lIRy6IF9GZcIrDK2KGItfor8IQwQa+yM2ERgHM1MmHxaxpTzk0/Jp42lNfA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@radix-ui/primitive": { + "version": "1.1.7", + "resolved": "https://registry.npmjs.org/@radix-ui/primitive/-/primitive-1.1.7.tgz", + "integrity": "sha512-rqWnm76nYT8HoNNqEjpgJ7Pw/DrBj5iBTrmEPo6HTX5+VJyBNOqTdv4g89G63HuR5g0AaENoAcH7Is5fF2kZ8Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/@radix-ui/react-accessible-icon": { + "version": "1.1.15", + "resolved": "https://registry.npmjs.org/@radix-ui/react-accessible-icon/-/react-accessible-icon-1.1.15.tgz", + "integrity": "sha512-WTQwcAvQf5sOcuUyi90lKPbhwcvQ+j55cjrSmeaN+L2vKU3DooOvlKw2MDeiJ5IkV5N905KW0/fGojKOBhD11A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-visually-hidden": "1.2.11" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-accordion": { + "version": "1.2.20", + "resolved": "https://registry.npmjs.org/@radix-ui/react-accordion/-/react-accordion-1.2.20.tgz", + "integrity": "sha512-jDhG9FvAEnlhnjrsINbNXcUa4G+L1KqSkJSunkbKEzFRcAb52jvM0PjPxPRvhe1HNc5F5yc0yzzWeeqlH4yBIg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-collapsible": "1.1.20", + "@radix-ui/react-collection": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-controllable-state": "1.2.6" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-alert-dialog": { + "version": "1.1.23", + "resolved": "https://registry.npmjs.org/@radix-ui/react-alert-dialog/-/react-alert-dialog-1.1.23.tgz", + "integrity": "sha512-VAYOiQRqj3GPpYJE0I9J+X8Ip05cyVlNdKOFeiGS2Ou1HHGfpl0BxOyZm6nmVDyU+W+NF3/XLzmjHmVGydhwgA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-dialog": "1.1.23", + "@radix-ui/react-primitive": "2.1.10" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-arrow": { + "version": "1.1.15", + "resolved": "https://registry.npmjs.org/@radix-ui/react-arrow/-/react-arrow-1.1.15.tgz", + "integrity": "sha512-v4zggRcjadnI+ClKDuijlQEW4tw3NoaeHc/PwpKnLoLLKNUG4InLegkstooLcRIUWCs+8L22dGURCVuFfOKfnA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-primitive": "2.1.10" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-aspect-ratio": { + "version": "1.1.15", + "resolved": "https://registry.npmjs.org/@radix-ui/react-aspect-ratio/-/react-aspect-ratio-1.1.15.tgz", + "integrity": "sha512-fy+dyVR+90nelK8rqIznFlxzx7uPcGbhxH8Nfr2bHb4UfSe+e3hklOC0luK0hDwVwnRX7xTRySpsrQVeW+/oNQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-primitive": "2.1.10" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-avatar": { + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/@radix-ui/react-avatar/-/react-avatar-1.2.6.tgz", + "integrity": "sha512-4ULOTJ/mqy2hT9GlWa/MFHxHSvH3nJzHnZM1waNsc5Bonv7i70aNenghXmD97S6OJ81ekXONGGt4nT1r0PfEdA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-callback-ref": "1.1.4", + "@radix-ui/react-use-is-hydrated": "0.1.3", + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-checkbox": { + "version": "1.3.11", + "resolved": "https://registry.npmjs.org/@radix-ui/react-checkbox/-/react-checkbox-1.3.11.tgz", + "integrity": "sha512-Gnptr9pDDQxD3hgq2dtPbtrp/c2qH1mBwIzw3X/ivrMb2e1t0jMTi606fVEqFPaQR1ggXIVQWKj3P2WW9v7zGQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-size": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-collapsible": { + "version": "1.1.20", + "resolved": "https://registry.npmjs.org/@radix-ui/react-collapsible/-/react-collapsible-1.1.20.tgz", + "integrity": "sha512-mcGesGplBnzN2sbvJETzpCNfSMyPnb29q1GRLU+Ib7bJrpIG2ywmRoh2V5VbA2uNvKikKUlVbAPks7JDjz4A8Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-collection": { + "version": "1.1.15", + "resolved": "https://registry.npmjs.org/@radix-ui/react-collection/-/react-collection-1.1.15.tgz", + "integrity": "sha512-9W+B9NPF0NaaPh/1NJd3+KqsnlLqU9H7T2rvww+fp+T/evVXdNAyYcnfRQZFOjkR1ajQp3yORlqnI8soawLvNA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-slot": "1.3.3" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-compose-refs": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@radix-ui/react-compose-refs/-/react-compose-refs-1.1.5.tgz", + "integrity": "sha512-+48PbAAbq3didjJxa+OaWY2ZwgAKsNiRGyeHKszblZMQ+kcpd9pAaT11cMkGEie0vsOi3QdeTE6d5Fe3Gn61kA==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-context": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@radix-ui/react-context/-/react-context-1.2.2.tgz", + "integrity": "sha512-RHCUGwKHDr0hDGg4X7ma4JG4/+12qxw8rkh5QKdDldlCvtja6nUx1Ef/8HVrJze81lEsgLQlqjzjGNHantgnQA==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-context-menu": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/@radix-ui/react-context-menu/-/react-context-menu-2.3.7.tgz", + "integrity": "sha512-CtXP35dxaB5T3zXSd+E3uHe/QpXcpYnZmxp6OaIbfthtfW4wyb77M23BG+bwIJDtsMwEP/YssdsmNyZu7jhWew==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-menu": "2.1.24", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-controllable-state": "1.2.6" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-dialog": { + "version": "1.1.23", + "resolved": "https://registry.npmjs.org/@radix-ui/react-dialog/-/react-dialog-1.1.23.tgz", + "integrity": "sha512-Ksw4WeROkO4rC9k/onilX/Ao2Cr1ku1unMNH+XSCcP4jSXYu7HDsg9n4ojMjVb22XpYjAQ9qfrFlVbru1vXDUA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-dismissable-layer": "1.1.19", + "@radix-ui/react-focus-guards": "1.1.6", + "@radix-ui/react-focus-scope": "1.1.16", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-portal": "1.1.17", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-slot": "1.3.3", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-layout-effect": "1.1.4", + "aria-hidden": "^1.2.4", + "react-remove-scroll": "^2.7.2" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-direction": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-direction/-/react-direction-1.1.4.tgz", + "integrity": "sha512-5pzg4FGQNpExhnhT2zlrP1wZFaYCd1K0nYWoFAdcYoYK868IEigqMX3B3f8yIoRlAhAeDWciLI6ZdCKHF9P4Vg==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-dismissable-layer": { + "version": "1.1.19", + "resolved": "https://registry.npmjs.org/@radix-ui/react-dismissable-layer/-/react-dismissable-layer-1.1.19.tgz", + "integrity": "sha512-8g4pfOL9HoKKLWGiypT+dphVqjFfmcXO5GBnhsG6zI+lxAx/8feQpr+1LSN8Re3hiZ+XkLNS4O9ztK11/LzQ6w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-callback-ref": "1.1.4", + "@radix-ui/react-use-effect-event": "0.0.5" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-dropdown-menu": { + "version": "2.1.24", + "resolved": "https://registry.npmjs.org/@radix-ui/react-dropdown-menu/-/react-dropdown-menu-2.1.24.tgz", + "integrity": "sha512-geq8l2rJkxvkXsT9RMgtUE3P8pITFpTsvYpbySi1IH4fZEABD/Gp85myayFgxk0ktljGMJnCbeFkyTusvSvv7g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-menu": "2.1.24", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-controllable-state": "1.2.6" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-focus-guards": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/@radix-ui/react-focus-guards/-/react-focus-guards-1.1.6.tgz", + "integrity": "sha512-RNOJjfZMTyBM6xYmV3IVGXkPjIhcBAuv48POevAXwrGJhkWZ9p1rFoIS1JFooPuT193AZmRsCPhpoVJxx6OPoQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-focus-scope": { + "version": "1.1.16", + "resolved": "https://registry.npmjs.org/@radix-ui/react-focus-scope/-/react-focus-scope-1.1.16.tgz", + "integrity": "sha512-wmRZ2WWLvmt6KHy2rNPOdPUjwq5xOHY02+m+udwJTn0aNIox/rkskAvJTyTLGhPK6KgrUjlJUJpgmx/+wFiFIQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-callback-ref": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-form": { + "version": "0.1.16", + "resolved": "https://registry.npmjs.org/@radix-ui/react-form/-/react-form-0.1.16.tgz", + "integrity": "sha512-Q4TLEn2A7TAypxwmd6R9EwrlXDvkfYSDMrq9/887AXAGh+G1rH+kYJKSTv+Si9Y0JPKTwKYv6PviAJosysNimA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-label": "2.1.15", + "@radix-ui/react-primitive": "2.1.10" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-hover-card": { + "version": "1.1.23", + "resolved": "https://registry.npmjs.org/@radix-ui/react-hover-card/-/react-hover-card-1.1.23.tgz", + "integrity": "sha512-H8qONfZd3ltrU3+jHCIgITbWo6e1iTKvP9DHdrvYbX48ooRM5FjEDTn16AMwdfuOGkWdZEhpl3PLL/Wk/AnHDQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-dismissable-layer": "1.1.19", + "@radix-ui/react-popper": "1.3.7", + "@radix-ui/react-portal": "1.1.17", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-controllable-state": "1.2.6" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-id": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-id/-/react-id-1.1.4.tgz", + "integrity": "sha512-TMQp2llA+RYn7JcjnrMnz7wN4pcVttPZnRZo52PLQsoLVKzNlVwUeHmfePgTgRluXFvlD3GD5g5MOVVTJCO0qA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-label": { + "version": "2.1.15", + "resolved": "https://registry.npmjs.org/@radix-ui/react-label/-/react-label-2.1.15.tgz", + "integrity": "sha512-o/rdYEwZTTo5tjknnPeyQFU45kUC4i/XyeDPP+HGyi6XqpOP6Zf5Ya5vh/Yfe9Id5JiuWnnAx2XqIeD3UYZt0g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-primitive": "2.1.10" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-menu": { + "version": "2.1.24", + "resolved": "https://registry.npmjs.org/@radix-ui/react-menu/-/react-menu-2.1.24.tgz", + "integrity": "sha512-uW7RVuU6Lp/ZtfeY4b3kL32zccgEWvPv1+cf17ubYzHa9cL8AHokmk36cG/XEiH/smbQvumnieXX9j/e9RqJWA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-collection": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-dismissable-layer": "1.1.19", + "@radix-ui/react-focus-guards": "1.1.6", + "@radix-ui/react-focus-scope": "1.1.16", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-popper": "1.3.7", + "@radix-ui/react-portal": "1.1.17", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-roving-focus": "1.1.19", + "@radix-ui/react-slot": "1.3.3", + "@radix-ui/react-use-callback-ref": "1.1.4", + "aria-hidden": "^1.2.4", + "react-remove-scroll": "^2.7.2" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-menubar": { + "version": "1.1.24", + "resolved": "https://registry.npmjs.org/@radix-ui/react-menubar/-/react-menubar-1.1.24.tgz", + "integrity": "sha512-eeVs0vf7cuqXaM0qLQCPcufImiJNVBXdJDLu7ZGYl2732UH23Qat/foNGrr6vYV3/DdTsBqASoggUFgH14OcZA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-collection": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-menu": "2.1.24", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-roving-focus": "1.1.19", + "@radix-ui/react-use-controllable-state": "1.2.6" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-navigation-menu": { + "version": "1.2.22", + "resolved": "https://registry.npmjs.org/@radix-ui/react-navigation-menu/-/react-navigation-menu-1.2.22.tgz", + "integrity": "sha512-ou7iLEJ+yrhQndkkA4U21XIdS/CS45F4iXIkTZcb6/Ne9EMsOuDudVmCwmDnfFZZ+y1FZqXRNSIgBy+YMvZVZg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-collection": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-dismissable-layer": "1.1.19", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-callback-ref": "1.1.4", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-layout-effect": "1.1.4", + "@radix-ui/react-use-previous": "1.1.4", + "@radix-ui/react-visually-hidden": "1.2.11" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-one-time-password-field": { + "version": "0.1.16", + "resolved": "https://registry.npmjs.org/@radix-ui/react-one-time-password-field/-/react-one-time-password-field-0.1.16.tgz", + "integrity": "sha512-Tj9P6ntAJEw52oq/F0AGknXR4XncxEt7XU47O3xJQOiWfLzEy3d9gtgKfvjSzGxzHkfL+VzvxGu2KTFsloJqXw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/number": "1.1.3", + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-collection": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-roving-focus": "1.1.19", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-effect-event": "0.0.5", + "@radix-ui/react-use-is-hydrated": "0.1.3", + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-password-toggle-field": { + "version": "0.1.11", + "resolved": "https://registry.npmjs.org/@radix-ui/react-password-toggle-field/-/react-password-toggle-field-0.1.11.tgz", + "integrity": "sha512-4gvFnmDXu3dgj21CqsufzIameRvlRd4SBqaWhcrlrNhRo0Y5i/49AmRJYe1fdAM3G2VNBbmin4b0D6cdQocwgw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-effect-event": "0.0.5", + "@radix-ui/react-use-is-hydrated": "0.1.3" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-popover": { + "version": "1.1.23", + "resolved": "https://registry.npmjs.org/@radix-ui/react-popover/-/react-popover-1.1.23.tgz", + "integrity": "sha512-mw58MrBlyHWFisTOYignD0vf/3gdcgAR+9of1s9G/38CbFiUwH1nCDkc0AUM9IrXFgN5Ue8n45j9WCgyM1sbiQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-dismissable-layer": "1.1.19", + "@radix-ui/react-focus-guards": "1.1.6", + "@radix-ui/react-focus-scope": "1.1.16", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-popper": "1.3.7", + "@radix-ui/react-portal": "1.1.17", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-slot": "1.3.3", + "@radix-ui/react-use-controllable-state": "1.2.6", + "aria-hidden": "^1.2.4", + "react-remove-scroll": "^2.7.2" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-popper": { + "version": "1.3.7", + "resolved": "https://registry.npmjs.org/@radix-ui/react-popper/-/react-popper-1.3.7.tgz", + "integrity": "sha512-UsJrrd7w4wuKKTdvd/DNERVlwSlUcyXzjhyDwBk+3aPOsCjOY6ZSbxuw8E6lZTjjfP8Cpd0J8VVkrYUWyGYXyg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@floating-ui/react-dom": "^2.0.0", + "@radix-ui/react-arrow": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-callback-ref": "1.1.4", + "@radix-ui/react-use-layout-effect": "1.1.4", + "@radix-ui/react-use-rect": "1.1.4", + "@radix-ui/react-use-size": "1.1.4", + "@radix-ui/rect": "1.1.3" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-portal": { + "version": "1.1.17", + "resolved": "https://registry.npmjs.org/@radix-ui/react-portal/-/react-portal-1.1.17.tgz", + "integrity": "sha512-vKQLcWypUnwZVvfV7UkGahH2g6ySe8M8R+zYBwPrv5byZ9QAW6cQVvNKo7GgmD+p8aYb6D9JBuvy8/WhOno2wQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-presence": { + "version": "1.1.10", + "resolved": "https://registry.npmjs.org/@radix-ui/react-presence/-/react-presence-1.1.10.tgz", + "integrity": "sha512-3wyzCQ6+ubRA+D4uv9m95JYLXxmOHp05qjrkjeA7uKHHtjpPggQzc6DAb0URl7j67oR0K2foO4ip27TiX037Bw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-primitive": { + "version": "2.1.10", + "resolved": "https://registry.npmjs.org/@radix-ui/react-primitive/-/react-primitive-2.1.10.tgz", + "integrity": "sha512-MucOnzh6hR5mid6VpkbglRAMYMjKLqRnGBbjXkzjK52fuQDd1qbkx78a5P40mkcnVXJdEVxm26E9OPAiUq7nBg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-slot": "1.3.3" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-progress": { + "version": "1.1.16", + "resolved": "https://registry.npmjs.org/@radix-ui/react-progress/-/react-progress-1.1.16.tgz", + "integrity": "sha512-5XnomAsoZZCY+KNTxbIghpGqPruZvKFNlvcAljVAOdDRDsH4/OZQxhtwo5wdtoDM5R6MhJBb2sPnDuRFep3lzg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-primitive": "2.1.10" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-radio-group": { + "version": "1.4.7", + "resolved": "https://registry.npmjs.org/@radix-ui/react-radio-group/-/react-radio-group-1.4.7.tgz", + "integrity": "sha512-cgYFEkntCxppHZgtSZ+7vh0wbZQ+IC7PPMw8DSnRG27B6kDd32/Zw0OJt7dGDigCoprMuWHjg2PvUn3PYvPFoQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-roving-focus": "1.1.19", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-size": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-roving-focus": { + "version": "1.1.19", + "resolved": "https://registry.npmjs.org/@radix-ui/react-roving-focus/-/react-roving-focus-1.1.19.tgz", + "integrity": "sha512-V9jI6hDjT7l3jsCQD9bLNvDLM3tH/gdbOTp7Tefp3hbbgCGQoK7tUvrWiRlcoBHIZ809ElXwNQwVo0B98LuTXQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-collection": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-callback-ref": "1.1.4", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-is-hydrated": "0.1.3", + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-scroll-area": { + "version": "1.2.18", + "resolved": "https://registry.npmjs.org/@radix-ui/react-scroll-area/-/react-scroll-area-1.2.18.tgz", + "integrity": "sha512-Zn5Cd171wxsO3Dfg8HaW6RifTb9CYTKQJHs/G4+LN1GfmJpaQMZQyQxMprVPHpaz7QY4l9BxK2JwQuzHsXC8nA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/number": "1.1.3", + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-callback-ref": "1.1.4", + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-select": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/@radix-ui/react-select/-/react-select-2.3.7.tgz", + "integrity": "sha512-WFGImkmbzcfxeIwq/+4HvRN0pizBwbwQUED4I13ezQsDdfl38ZntN6TmR8XaSzPBqoCToe8rF75j6NPNDSzhbg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/number": "1.1.3", + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-collection": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-dismissable-layer": "1.1.19", + "@radix-ui/react-focus-guards": "1.1.6", + "@radix-ui/react-focus-scope": "1.1.16", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-popper": "1.3.7", + "@radix-ui/react-portal": "1.1.17", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-slot": "1.3.3", + "@radix-ui/react-use-callback-ref": "1.1.4", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-layout-effect": "1.1.4", + "@radix-ui/react-use-previous": "1.1.4", + "@radix-ui/react-visually-hidden": "1.2.11", + "aria-hidden": "^1.2.4", + "react-remove-scroll": "^2.7.2" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-separator": { + "version": "1.1.15", + "resolved": "https://registry.npmjs.org/@radix-ui/react-separator/-/react-separator-1.1.15.tgz", + "integrity": "sha512-jOLO4lssEzWpoDu7G+Ze4VjwMRUBt291pnZD0gmalREZipnTX3wadQo7Fy48GCTfe14/YRN6rw/rOJqrE85Wxw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-primitive": "2.1.10" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-slider": { + "version": "1.4.7", + "resolved": "https://registry.npmjs.org/@radix-ui/react-slider/-/react-slider-1.4.7.tgz", + "integrity": "sha512-mTSLf1GC/C0moWjTbvCM6Qn/gBjvlFt1azuWF2v7MN5C3Zq2U2J2lN3ZEYkpujuOU5Ro7A28wkviSxaKnG0BYg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/number": "1.1.3", + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-collection": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-layout-effect": "1.1.4", + "@radix-ui/react-use-previous": "1.1.4", + "@radix-ui/react-use-size": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-slot": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@radix-ui/react-slot/-/react-slot-1.3.3.tgz", + "integrity": "sha512-qx7oqnYbxnK9kYI9m317qmFmEgo6ywqWvbTogdj7cL9p3/yx4M48p7Rnw5z3H890cL/ow/EeWJsuTykeZVXP5Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-compose-refs": "1.1.5" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-switch": { + "version": "1.3.7", + "resolved": "https://registry.npmjs.org/@radix-ui/react-switch/-/react-switch-1.3.7.tgz", + "integrity": "sha512-48tB/4dn2UVLBCYhTu9AuR63IHl73l/qLbLgxd86noTUor4/K4LFDAcYjK+isP5313qxaFpjPVogE7+Y0/V3Kw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-size": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-tabs": { + "version": "1.1.21", + "resolved": "https://registry.npmjs.org/@radix-ui/react-tabs/-/react-tabs-1.1.21.tgz", + "integrity": "sha512-UKxJlZid7FVtsk/WTxj4i4uSEgj2Au+KBbS7SQyTlzMhhn+86Cz3tISZdTa87bfEfcuvZezf2ZsxD4xuEKtkog==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-roving-focus": "1.1.19", + "@radix-ui/react-use-controllable-state": "1.2.6" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-toast": { + "version": "1.2.23", + "resolved": "https://registry.npmjs.org/@radix-ui/react-toast/-/react-toast-1.2.23.tgz", + "integrity": "sha512-ofhyAsYaocRGOs/n0XWdUOSVzEAG6BfrMVM8z0c0kLEWY38w/0WuMFPTJP/HVaZPYkMvHZoKIIhNcjbTCBILPg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-collection": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-dismissable-layer": "1.1.19", + "@radix-ui/react-portal": "1.1.17", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-callback-ref": "1.1.4", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-layout-effect": "1.1.4", + "@radix-ui/react-visually-hidden": "1.2.11" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-toggle": { + "version": "1.1.18", + "resolved": "https://registry.npmjs.org/@radix-ui/react-toggle/-/react-toggle-1.1.18.tgz", + "integrity": "sha512-7lonPlKfSacd20GlOBx2ltuVKz9oqWYZz+oMQyOltw6t1y2nyftj2ZmwwUHYn49kqfDWcp8dNZm5NgV+5Z+mug==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-controllable-state": "1.2.6" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-toggle-group": { + "version": "1.1.19", + "resolved": "https://registry.npmjs.org/@radix-ui/react-toggle-group/-/react-toggle-group-1.1.19.tgz", + "integrity": "sha512-OtnwuSVjd1Ofi+AdnvhsjQdyuhCDwYs1w9RyB5BN/OavXOVQo42SYqQjwUnbPnaiPFBpQ9aX70dWeee+v2oBLA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-roving-focus": "1.1.19", + "@radix-ui/react-toggle": "1.1.18", + "@radix-ui/react-use-controllable-state": "1.2.6" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-toolbar": { + "version": "1.1.19", + "resolved": "https://registry.npmjs.org/@radix-ui/react-toolbar/-/react-toolbar-1.1.19.tgz", + "integrity": "sha512-Ph0IvtYw4VB12ZnZg+YtrGs8yJQsnizwo/zu0R4Y/nWugtJzA7Pg1eWeuDR9+LSqn+xjamss+UOSOJJJ4gx8jw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-roving-focus": "1.1.19", + "@radix-ui/react-separator": "1.1.15", + "@radix-ui/react-toggle-group": "1.1.19" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-tooltip": { + "version": "1.2.16", + "resolved": "https://registry.npmjs.org/@radix-ui/react-tooltip/-/react-tooltip-1.2.16.tgz", + "integrity": "sha512-6EamKFRRnlpdadndbZ6LMwycfwkwPte1B42hs6QA0gYhjaOKqW4PZ4pjaW9UrlDX5eVt/OjncE7BFTPL5nmZhg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-dismissable-layer": "1.1.19", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-popper": "1.3.7", + "@radix-ui/react-portal": "1.1.17", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-slot": "1.3.3", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-layout-effect": "1.1.4", + "@radix-ui/react-visually-hidden": "1.2.11" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-callback-ref": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-callback-ref/-/react-use-callback-ref-1.1.4.tgz", + "integrity": "sha512-R6OUY2e2fA6Yn6s+VSx5KBV6Nx8LQEhu+cz7LCej18rQ1HLyg9PSC9jP/ZNx0o6FAIK9c0F1kHylzSxKsdlkrQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-controllable-state": { + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-controllable-state/-/react-use-controllable-state-1.2.6.tgz", + "integrity": "sha512-uEQJGT97ZA/TgP/Hydw47lHu+/vQj6z/0jA+WeTbK1o9Rx45GImjpD0tc3W5ad3D6XTSR6e1yEO0FvGq6WQfVQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-use-effect-event": "0.0.5", + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-effect-event": { + "version": "0.0.5", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-effect-event/-/react-use-effect-event-0.0.5.tgz", + "integrity": "sha512-7cshFL8HGS/7HEiHH+9kL9HBwp2sa9yX18Knwek6KYWmXwM7pegMgta2AXMQKI+rq3JnfSj9x8wYqFMTdG1Jgg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-escape-keydown": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-escape-keydown/-/react-use-escape-keydown-1.1.5.tgz", + "integrity": "sha512-ge3ipobwSXTj4JyVtswQ7qZj0ZHdtbGuOno/LrgAAeSxtsJ6Vs4Gz5IkPH2bmqpjcLUFoqGhA/mueuIf63UXlA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-use-callback-ref": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-is-hydrated": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-is-hydrated/-/react-use-is-hydrated-0.1.3.tgz", + "integrity": "sha512-umO/aJ+82CpOnhDZUTbILCQf7kU/g0iv+oGs/Q8jw7IkhWBzaEP4sA268PhFAJTFetbwp3ICc6ktpI4TqtxcIw==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-layout-effect": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-layout-effect/-/react-use-layout-effect-1.1.4.tgz", + "integrity": "sha512-K20DkRkUwDnxEYMBPcg3Y6voLkEy5p5QQmszZgLngKKiC7dzBR/aEuK3w1qlx2JWDUNH6FluahYdgR3BP+QbYw==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-previous": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-previous/-/react-use-previous-1.1.4.tgz", + "integrity": "sha512-XoSLhbRbqxFtgJoi2fNHA3C6pDlY34x508vUpUGoFZfvePfHXHbE1lC4FYFMnJWgiCRroSTw6fOsXQoVS9RwZg==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-rect": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-rect/-/react-use-rect-1.1.4.tgz", + "integrity": "sha512-cSOCh6JlkmfjLyNcLiu2nB4v+nm+dkZ+Q5KHWk/soo4U7ZLiEQFKHK9/YmtBHjfCEaU43IBKQOc4/uJmCaiCTQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/rect": "1.1.3" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-size": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-size/-/react-use-size-1.1.4.tgz", + "integrity": "sha512-D3anSY15EJoxrihpsXI6SMrmmonnQtR2ni7arO+Lfdg3O95b9hNXxONk8jA5C8ANdF/h5HMAxejgs8PWJ6rlhw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-visually-hidden": { + "version": "1.2.11", + "resolved": "https://registry.npmjs.org/@radix-ui/react-visually-hidden/-/react-visually-hidden-1.2.11.tgz", + "integrity": "sha512-NFS86RYYZb4/exihaESBGOpMJFz8MGLAfu3mOBSGByVnVPC9JPASfYubxd/8KbkQK0sYAv8lVQDEQukDX/qXvQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/react-primitive": "2.1.10" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/rect": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@radix-ui/rect/-/rect-1.1.3.tgz", + "integrity": "sha512-JtyZR+mqgBibTo8xea3B6ZRmzZiM/YeVBtUkas6zMuXjAlfIFIW2FgqeM9eLyvEaYX66vr6DJMK+4U6LV0KhNw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@rolldown/binding-android-arm-eabi": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm-eabi/-/binding-android-arm-eabi-1.2.8.tgz", + "integrity": "sha512-tN5aztYkKCte4i5SIrrz5yK/HMjEuCqCSCJa418jOV8tZ1cBY3YF2otxB1ktPxzsLA1BeTqwapK0bfjxNvHJVw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-android-arm64": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.2.8.tgz", + "integrity": "sha512-dIYTWl9XprMUiQFoc55KUyk/oS8SKYH3zFl0LTR7RT0Xj4hgSVyuJcroH8JUu8RcpF8fTB6E0aOwCkZoYPcDSQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-arm64": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.2.8.tgz", + "integrity": "sha512-PCSDQGXD2IyTEFrcgPyBM8jJuGmrbCMuoIOXdbEGVemruKACXoLQJrb+A45Z0L5t1RQkdfJprAYPkikbh7dzdA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-x64": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.2.8.tgz", + "integrity": "sha512-Uk7lRsGhPFHVX/sAUC6D5H9Ol30dFHd6iquokll2th3LpdJ3F5CzQB+7DHn0Ri2mG+U7k2zXiPHDrwZenXhwSA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-freebsd-x64": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.2.8.tgz", + "integrity": "sha512-DjszaTEVogPqA5bYzsEeqDCQxbcp2fexQwKcRspYji2yzR68fCf+e4fx6kBSRDwX5/brZaHw/hWS9+A/+/w9sQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm-gnueabihf": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.2.8.tgz", + "integrity": "sha512-zmwa7FTmdzB6aaEEuuls18H6Ap5JmJPSoPTuXixeJZV6tG40SyLkApQtz1g8ptZtiEKqj9OM0oNLPh1AgvE31Q==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-gnu": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.2.8.tgz", + "integrity": "sha512-KdYQDPHwJVnbFwdTGMgxsI9SqblBlz6STGM+w1We/d5B8OWWidYH0MwkU/uA1wM5fIpO2MkOVxXrNzzuZhw9ew==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-musl": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.2.8.tgz", + "integrity": "sha512-jFJTifHnNPY+yzOoNZQfSIysrVyXzEQPhPnOUjmD1bcQGHH6s7c8cViKWar8YplQImE5N9JRqMCLrM2CdxOrZA==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-ppc64-gnu": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.2.8.tgz", + "integrity": "sha512-FhiOziBDWPBjbcmRzfLyIJnaP7AVMFXT7YCXPjXxj7wKU3vx24RjrCNN/zjvVa+N2vVoHJwCoUBvsrN/DG3zIA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-s390x-gnu": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.2.8.tgz", + "integrity": "sha512-WnHfADMzOV2Y55wlx1hzzQnar/wDt/VdvWSD99r18Mz9ylNieIGOkRx3UV21h7m/eJvjySYJkO26VvGNFkwsIQ==", + "cpu": [ + "s390x" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-gnu": { + "version": "1.2.8", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-musl": { + "version": "1.2.8", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-openharmony-arm64": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.2.8.tgz", + "integrity": "sha512-637Ke4kWSy6rp9cxQ9gMOXlxPgIw/c1beASV4M//3+9I4uwBVOOl74G+e3zyU3u19U7RkRl/HuewixZ/Z6+Rjg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-win32-arm64-msvc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.2.8.tgz", + "integrity": "sha512-xWBkPOF1Q9k/Gv1nQXnVdLxKu74jXppuOM4Z3mnypVUJJJwLsMl7hNJGRAUJoG8A5MgOI1ACKM+wBFxSJzKy4A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-win32-x64-msvc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.2.8.tgz", + "integrity": "sha512-uz2ZvfgXbxqNwijjjbxrnvALwpyODDcgc1T1N8N3rf/DXKQmaFwmB4LX4yyjggpwN2obdQLb2rgirX5ffCWYng==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/pluginutils": { + "version": "1.0.1", + "dev": true, + "license": "MIT" + }, + "node_modules/@standard-schema/spec": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", + "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/bun": { + "version": "1.4.2", + "dev": true, + "license": "MIT", + "dependencies": { + "bun-types": "1.4.2" + } + }, + "node_modules/@types/node": { + "version": "26.5.1", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~8.9.0" + } + }, + "node_modules/@types/react": { + "version": "19.3.0", + "dev": true, + "license": "MIT", + "dependencies": { + "csstype": "^3.2.2" + } + }, + "node_modules/@types/react-dom": { + "version": "19.3.0", + "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-19.3.0.tgz", + "integrity": "sha512-ZI7bU42mZXXKHn/qNLEw2IrbiINU7X5+vfgdixBHkCNpYWXjKgfQ/P+uyGb5CjOLB9UcnTeg3rylQtV2hym44Q==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "^19.3.0" + } + }, + "node_modules/@types/whatwg-mimetype": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@types/whatwg-mimetype/-/whatwg-mimetype-3.0.2.tgz", + "integrity": "sha512-c2AKvDT8ToxLIOUlN51gTiHXflsfIFisS4pO7pDPoKouJCESkhZnEy623gwP9laCy5lnLDAw1vAzu2vM2YLOrA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/ws": { + "version": "8.18.1", + "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.18.1.tgz", + "integrity": "sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@vercel/oidc": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/@vercel/oidc/-/oidc-3.2.0.tgz", + "integrity": "sha512-UycprH3T6n3jH0k44NHMa7pnFHGu/N05MjojYr+Mc6I7obkoLIJujSWwin1pCvdy/eOxrI/l3uDLQsmcrOb4ug==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">= 20" + } + }, + "node_modules/@workflow/serde": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@workflow/serde/-/serde-4.1.0.tgz", + "integrity": "sha512-pav4F2BoirECWR7Nf1TKt+2eETcBj7jj4cBefQ8VXQCA6NPkaKeLfj/zMgi+3zYV5ZIBT4GuUiphsj0/b9hPQQ==", + "dev": true, + "license": "Apache-2.0" + }, + "node_modules/ai": { + "version": "7.0.99", + "resolved": "https://registry.npmjs.org/ai/-/ai-7.0.99.tgz", + "integrity": "sha512-Ov+3j/nSajaVH5hO8C94wN9wisG5tAJ8HWsLV9d/+nlzrRGUh3oYO3Kp1fRyUNWjAWLSpGLHLPaHCdfxb307Vg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/gateway": "4.0.80", + "@ai-sdk/provider": "4.0.14", + "@ai-sdk/provider-utils": "5.0.40" + }, + "engines": { + "node": ">=22" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/ansis": { + "version": "4.4.0", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=14" + } + }, + "node_modules/aria-hidden": { + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/aria-hidden/-/aria-hidden-1.2.6.tgz", + "integrity": "sha512-ik3ZgC9dY/lYVVM++OISsaYDeg1tb0VtP5uL3ouh1koGOaUMDPpbFIei4JkFimWUFPn90sbMNMXQAIVOlnYKJA==", + "dev": true, + "license": "MIT", + "dependencies": { + "tslib": "^2.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/assistant-cloud": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/assistant-cloud/-/assistant-cloud-0.2.0.tgz", + "integrity": "sha512-LMvPaufIfZ0dpByMsNeuPFY6UeKQ1oiSBktUJuhjL/2eTZLSzzjIWumiLALlpdvUZbbJCRHy8j0ZgzlZJJlEfQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "assistant-stream": "^0.3.42" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.9.0", + "@opentelemetry/exporter-trace-otlp-http": ">=0.200.0", + "@opentelemetry/sdk-trace-base": "^2.1.0" + }, + "peerDependenciesMeta": { + "@opentelemetry/api": { + "optional": true + }, + "@opentelemetry/exporter-trace-otlp-http": { + "optional": true + }, + "@opentelemetry/sdk-trace-base": { + "optional": true + } + } + }, + "node_modules/assistant-stream": { + "version": "0.3.42", + "resolved": "https://registry.npmjs.org/assistant-stream/-/assistant-stream-0.3.42.tgz", + "integrity": "sha512-5dQxc7XX92LuJ8wuHaOi/vkHitz2DARABzoE364O7o4jiePWalDpDpqU9bN+bXTvnSiRrpesy6t3Z3oDqRkdQw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@standard-schema/spec": "^1.1.0", + "nanoid": "^6.0.1", + "secure-json-parse": "^4.1.0" + }, + "peerDependencies": { + "ioredis": "^5.10.1 || ^6.0.0", + "redis": "^5.12.1" + }, + "peerDependenciesMeta": { + "ioredis": { + "optional": true + }, + "redis": { + "optional": true + } + } + }, + "node_modules/ast-kit": { + "version": "2.2.0", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.28.5", + "pathe": "^2.0.3" + }, + "engines": { + "node": ">=20.19.0" + }, + "funding": { + "url": "https://github.com/sponsors/sxzz" + } + }, + "node_modules/birpc": { + "version": "2.9.0", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/buffer-image-size": { + "version": "0.6.4", + "resolved": "https://registry.npmjs.org/buffer-image-size/-/buffer-image-size-0.6.4.tgz", + "integrity": "sha512-nEh+kZOPY1w+gcCMobZ6ETUp9WfibndnosbpwB1iJk/8Gt5ZF2bhS6+B6bPYz424KtwsR6Rflc3tCz1/ghX2dQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + }, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/bun-types": { + "version": "1.4.2", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/cac": { + "version": "6.7.14", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/chokidar": { + "version": "4.0.3", + "dev": true, + "license": "MIT", + "dependencies": { + "readdirp": "^4.0.1" + }, + "engines": { + "node": ">= 14.16.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "dev": true, + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/csstype": { + "version": "3.2.3", + "dev": true, + "license": "MIT" + }, + "node_modules/debug": { + "version": "4.4.3", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/defu": { + "version": "6.1.7", + "dev": true, + "license": "MIT" + }, + "node_modules/dequal": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", + "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/detect-node-es": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/detect-node-es/-/detect-node-es-1.1.0.tgz", + "integrity": "sha512-ypdmJU/TbBby2Dxibuv7ZLW3Bs1QEmM7nHjEANfohJLvE0XVujisn1qPJcZxg+qDucsr+bP6fLD1rPS3AhJ7EQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/diff": { + "version": "8.0.4", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.3.1" + } + }, + "node_modules/dts-resolver": { + "version": "2.1.3", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=20.19.0" + }, + "funding": { + "url": "https://github.com/sponsors/sxzz" + }, + "peerDependencies": { + "oxc-resolver": ">=11.0.0" + }, + "peerDependenciesMeta": { + "oxc-resolver": { + "optional": true + } + } + }, + "node_modules/empathic": { + "version": "2.0.1", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14" + } + }, + "node_modules/entities": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/entities/-/entities-7.0.1.tgz", + "integrity": "sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/eventsource-parser": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.1.1.tgz", + "integrity": "sha512-EKN1vKAMcZ8MlYMpaNuxN6R9yakzH6uajHcHVTqWJzvu5pWw9DyhbP35HH8MVBQ+dZjAfDxk+A8NiR9KWaXiyQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/fdir": { + "version": "6.5.0", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/get-nonce": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-nonce/-/get-nonce-1.0.1.tgz", + "integrity": "sha512-FJhYRoDaiatfEkUK8HKlicmu/3SGFD51q3itKDGoSTysQJBnfOcxU5GxnhE1E6soB76MbT0MBtnKJuXyAx+96Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/get-tsconfig": { + "version": "4.14.3", + "dev": true, + "license": "MIT", + "dependencies": { + "resolve-pkg-maps": "^1.0.0" + }, + "funding": { + "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" + } + }, + "node_modules/happy-dom": { + "version": "20.14.5", + "resolved": "https://registry.npmjs.org/happy-dom/-/happy-dom-20.14.5.tgz", + "integrity": "sha512-x/RzkpWO40bTjIoT30iQtt64FLLmH/iRcUCN2X//bLx7H3ifkdfPXyqsro/OYtqzIAhiLMMA7mmiOR9C3NOKjQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": ">=20.0.0", + "@types/whatwg-mimetype": "^3.0.2", + "@types/ws": "^8.18.1", + "buffer-image-size": "^0.6.4", + "entities": "^7.0.1", + "whatwg-mimetype": "^3.0.0", + "ws": "^8.21.0" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/hookable": { + "version": "5.5.3", + "dev": true, + "license": "MIT" + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true, + "license": "ISC" + }, + "node_modules/jiti": { + "version": "2.7.0", + "dev": true, + "license": "MIT", + "bin": { + "jiti": "lib/jiti-cli.mjs" + } + }, + "node_modules/jsesc": { + "version": "3.1.0", + "dev": true, + "license": "MIT", + "bin": { + "jsesc": "bin/jsesc" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/json-schema": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz", + "integrity": "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==", + "dev": true, + "license": "(AFL-2.1 OR BSD-3-Clause)" + }, + "node_modules/ms": { + "version": "2.1.3", + "dev": true, + "license": "MIT" + }, + "node_modules/nanoid": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-6.0.1.tgz", + "integrity": "sha512-3wVS3i51pE2pi1k5FFL/95BGfVS0kSsvDVuGXHOtxox/TywUmtgq+3qiTOTbs9J7KfHaXPiN171k/A6dBnaXFw==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.js" + }, + "engines": { + "node": "^22 || ^24 || >=26" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/pathe": { + "version": "2.0.3", + "dev": true, + "license": "MIT" + }, + "node_modules/picomatch": { + "version": "4.0.7", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/pkce-challenge": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/pkce-challenge/-/pkce-challenge-5.0.1.tgz", + "integrity": "sha512-wQ0b/W4Fr01qtpHlqSqspcj3EhBvimsdh0KlHhH8HRZnMsEa0ea2fTULOXOS9ccQr3om+GcGRk4e+isrZWV8qQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/quansync": { + "version": "1.0.0", + "dev": true, + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/antfu" + }, + { + "type": "individual", + "url": "https://github.com/sponsors/sxzz" + } + ], + "license": "MIT" + }, + "node_modules/radix-ui": { + "version": "1.6.7", + "resolved": "https://registry.npmjs.org/radix-ui/-/radix-ui-1.6.7.tgz", + "integrity": "sha512-QBdhh1arIEUvPC0dQ5+nwWAxt7+N+oP/9jPwjJkGFoSk/sqxg32gJtSXGtFh8frAIcS6oC9cx2Q+7KYCQLOAeA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-accessible-icon": "1.1.15", + "@radix-ui/react-accordion": "1.2.20", + "@radix-ui/react-alert-dialog": "1.1.23", + "@radix-ui/react-arrow": "1.1.15", + "@radix-ui/react-aspect-ratio": "1.1.15", + "@radix-ui/react-avatar": "1.2.6", + "@radix-ui/react-checkbox": "1.3.11", + "@radix-ui/react-collapsible": "1.1.20", + "@radix-ui/react-collection": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-context-menu": "2.3.7", + "@radix-ui/react-dialog": "1.1.23", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-dismissable-layer": "1.1.19", + "@radix-ui/react-dropdown-menu": "2.1.24", + "@radix-ui/react-focus-guards": "1.1.6", + "@radix-ui/react-focus-scope": "1.1.16", + "@radix-ui/react-form": "0.1.16", + "@radix-ui/react-hover-card": "1.1.23", + "@radix-ui/react-label": "2.1.15", + "@radix-ui/react-menu": "2.1.24", + "@radix-ui/react-menubar": "1.1.24", + "@radix-ui/react-navigation-menu": "1.2.22", + "@radix-ui/react-one-time-password-field": "0.1.16", + "@radix-ui/react-password-toggle-field": "0.1.11", + "@radix-ui/react-popover": "1.1.23", + "@radix-ui/react-popper": "1.3.7", + "@radix-ui/react-portal": "1.1.17", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-progress": "1.1.16", + "@radix-ui/react-radio-group": "1.4.7", + "@radix-ui/react-roving-focus": "1.1.19", + "@radix-ui/react-scroll-area": "1.2.18", + "@radix-ui/react-select": "2.3.7", + "@radix-ui/react-separator": "1.1.15", + "@radix-ui/react-slider": "1.4.7", + "@radix-ui/react-slot": "1.3.3", + "@radix-ui/react-switch": "1.3.7", + "@radix-ui/react-tabs": "1.1.21", + "@radix-ui/react-toast": "1.2.23", + "@radix-ui/react-toggle": "1.1.18", + "@radix-ui/react-toggle-group": "1.1.19", + "@radix-ui/react-toolbar": "1.1.19", + "@radix-ui/react-tooltip": "1.2.16", + "@radix-ui/react-use-callback-ref": "1.1.4", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-effect-event": "0.0.5", + "@radix-ui/react-use-escape-keydown": "1.1.5", + "@radix-ui/react-use-is-hydrated": "0.1.3", + "@radix-ui/react-use-layout-effect": "1.1.4", + "@radix-ui/react-use-size": "1.1.4", + "@radix-ui/react-visually-hidden": "1.2.11" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/react": { + "version": "19.3.0", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/react-dom": { + "version": "19.3.0", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.3.0.tgz", + "integrity": "sha512-JDk8dgif51OjFoDE70+OT9ICyYr+69HlmihNwp1+Nsfbna3t5sIiCa9ZJktDmQ4/1b/rn26hIAR2uYXDMr5r0Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "scheduler": "^0.28.0" + }, + "peerDependencies": { + "react": "^19.3.0" + } + }, + "node_modules/react-remove-scroll": { + "version": "2.7.2", + "resolved": "https://registry.npmjs.org/react-remove-scroll/-/react-remove-scroll-2.7.2.tgz", + "integrity": "sha512-Iqb9NjCCTt6Hf+vOdNIZGdTiH1QSqr27H/Ek9sv/a97gfueI/5h1s3yRi1nngzMUaOOToin5dI1dXKdXiF+u0Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "react-remove-scroll-bar": "^2.3.7", + "react-style-singleton": "^2.2.3", + "tslib": "^2.1.0", + "use-callback-ref": "^1.3.3", + "use-sidecar": "^1.1.3" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/react-remove-scroll-bar": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/react-remove-scroll-bar/-/react-remove-scroll-bar-2.3.8.tgz", + "integrity": "sha512-9r+yi9+mgU33AKcj6IbT9oRCO78WriSj6t/cF8DWBZJ9aOGPOTEDvdUDz1FwKim7QXWwmHqtdHnRJfhAxEG46Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "react-style-singleton": "^2.2.2", + "tslib": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/react-style-singleton": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/react-style-singleton/-/react-style-singleton-2.2.3.tgz", + "integrity": "sha512-b6jSvxvVnyptAiLjbkWLE/lOnR4lfTtDAl+eUC7RZy+QQWc6wRzIV2CE6xBuMmDxc2qIihtDCZD5NPOFl7fRBQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "get-nonce": "^1.0.0", + "tslib": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/react-textarea-autosize": { + "version": "8.5.9", + "resolved": "https://registry.npmjs.org/react-textarea-autosize/-/react-textarea-autosize-8.5.9.tgz", + "integrity": "sha512-U1DGlIQN5AwgjTyOEnI1oCcMuEr1pv1qOtklB2l4nyMGbHzWrI0eFsYK0zos2YWqAolJyG0IWJaqWmWj5ETh0A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.20.13", + "use-composed-ref": "^1.3.0", + "use-latest": "^1.2.1" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, + "node_modules/readdirp": { + "version": "4.1.2", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 14.18.0" + }, + "funding": { + "type": "individual", + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/resolve-pkg-maps": { + "version": "1.0.0", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1" + } + }, + "node_modules/rolldown": { + "version": "1.2.8", + "dev": true, + "license": "MIT", + "dependencies": { + "@oxc-project/types": "=0.149.0", + "@rolldown/pluginutils": "^1.0.0" + }, + "bin": { + "rolldown": "bin/cli.mjs" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "optionalDependencies": { + "@rolldown/binding-android-arm-eabi": "1.2.8", + "@rolldown/binding-android-arm64": "1.2.8", + "@rolldown/binding-darwin-arm64": "1.2.8", + "@rolldown/binding-darwin-x64": "1.2.8", + "@rolldown/binding-freebsd-x64": "1.2.8", + "@rolldown/binding-linux-arm-gnueabihf": "1.2.8", + "@rolldown/binding-linux-arm64-gnu": "1.2.8", + "@rolldown/binding-linux-arm64-musl": "1.2.8", + "@rolldown/binding-linux-ppc64-gnu": "1.2.8", + "@rolldown/binding-linux-s390x-gnu": "1.2.8", + "@rolldown/binding-linux-x64-gnu": "1.2.8", + "@rolldown/binding-linux-x64-musl": "1.2.8", + "@rolldown/binding-openharmony-arm64": "1.2.8", + "@rolldown/binding-win32-arm64-msvc": "1.2.8", + "@rolldown/binding-win32-x64-msvc": "1.2.8" + } + }, + "node_modules/rolldown-plugin-dts": { + "version": "0.13.14", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/generator": "^7.28.0", + "@babel/parser": "^7.28.0", + "@babel/types": "^7.28.1", + "ast-kit": "^2.1.1", + "birpc": "^2.5.0", + "debug": "^4.4.1", + "dts-resolver": "^2.1.1", + "get-tsconfig": "^4.10.1" + }, + "engines": { + "node": ">=20.18.0" + }, + "funding": { + "url": "https://github.com/sponsors/sxzz" + }, + "peerDependencies": { + "@typescript/native-preview": ">=7.0.0-dev.20250601.1", + "rolldown": "^1.0.0-beta.9", + "typescript": "^5.0.0", + "vue-tsc": "^2.2.0 || ^3.0.0" + }, + "peerDependenciesMeta": { + "@typescript/native-preview": { + "optional": true + }, + "typescript": { + "optional": true + }, + "vue-tsc": { + "optional": true + } + } + }, + "node_modules/safe-content-frame": { + "version": "0.0.30", + "resolved": "https://registry.npmjs.org/safe-content-frame/-/safe-content-frame-0.0.30.tgz", + "integrity": "sha512-t8XO/59b+YaJCmCpJ3aTZ8TyB6LzG7VzCtUO3u8h0jUsK/wCvQPMro8ioCA6AQr0Ve8Cv9zunFp1VcFGm1aybg==", + "dev": true, + "license": "MIT" + }, + "node_modules/scheduler": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.28.0.tgz", + "integrity": "sha512-juorfCmIkIw8tT+p5BXSm6PJjQF/ycEYmKyzURCIt/RaZIhL+PulbQ9Yu2z1HdOJDdqDTlxA1+xKBmHXJsczAw==", + "dev": true, + "license": "MIT" + }, + "node_modules/secure-json-parse": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-4.1.0.tgz", + "integrity": "sha512-l4KnYfEyqYJxDwlNVyRfO2E4NTHfMKAWdUuA8J0yve2Dz/E/PdBepY03RvyJpssIpRFwJoCD55wA+mEDs6ByWA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/semver": { + "version": "7.8.5", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/swr": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/swr/-/swr-2.5.1.tgz", + "integrity": "sha512-BRw55e8r0B7SpDN20CAzoQAHl7y1yP7/Zt7oqUjMv0vSt2u2Xnkm88Ws+VypbV9BXHQVuSuyVq7zMjO16wSExw==", + "dev": true, + "license": "MIT", + "dependencies": { + "dequal": "^2.0.3", + "use-sync-external-store": "^1.6.0" + }, + "peerDependencies": { + "react": "^16.11.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, + "node_modules/throttleit": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/throttleit/-/throttleit-2.1.0.tgz", + "integrity": "sha512-nt6AMGKW1p/70DF/hGBdJB57B8Tspmbp5gfJ8ilhLnt7kkr2ye7hzD6NVG8GGErk2HWF34igrL2CXmNIkzKqKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/tinyexec": { + "version": "1.3.1", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/tinyglobby": { + "version": "0.2.17", + "dev": true, + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.4" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/tsdown": { + "version": "0.12.9", + "dev": true, + "license": "MIT", + "dependencies": { + "ansis": "^4.1.0", + "cac": "^6.7.14", + "chokidar": "^4.0.3", + "debug": "^4.4.1", + "diff": "^8.0.2", + "empathic": "^2.0.0", + "hookable": "^5.5.3", + "rolldown": "^1.0.0-beta.19", + "rolldown-plugin-dts": "^0.13.12", + "semver": "^7.7.2", + "tinyexec": "^1.0.1", + "tinyglobby": "^0.2.14", + "unconfig": "^7.3.2" + }, + "bin": { + "tsdown": "dist/run.mjs" + }, + "engines": { + "node": ">=18.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sxzz" + }, + "peerDependencies": { + "@arethetypeswrong/core": "^0.18.1", + "publint": "^0.3.0", + "typescript": "^5.0.0", + "unplugin-lightningcss": "^0.4.0", + "unplugin-unused": "^0.5.0" + }, + "peerDependenciesMeta": { + "@arethetypeswrong/core": { + "optional": true + }, + "publint": { + "optional": true + }, + "typescript": { + "optional": true + }, + "unplugin-lightningcss": { + "optional": true + }, + "unplugin-unused": { + "optional": true + } + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "dev": true, + "license": "0BSD" + }, + "node_modules/typescript": { + "version": "5.9.3", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/unconfig": { + "version": "7.5.0", + "dev": true, + "license": "MIT", + "dependencies": { + "@quansync/fs": "^1.0.0", + "defu": "^6.1.4", + "jiti": "^2.6.1", + "quansync": "^1.0.0", + "unconfig-core": "7.5.0" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/unconfig-core": { + "version": "7.5.0", + "dev": true, + "license": "MIT", + "dependencies": { + "@quansync/fs": "^1.0.0", + "quansync": "^1.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/undici": { + "version": "7.29.1", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.29.1.tgz", + "integrity": "sha512-RYONW2MeafgYlkVOKYKkA/Ag7BmXqgIWCa8t1m0JcxrQg9pI9lEqRhAOruOBCbAohOa/gkCF+iPi9hrgvTzu6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=20.18.1" + } + }, + "node_modules/undici-types": { + "version": "8.9.0", + "dev": true, + "license": "MIT" + }, + "node_modules/use-callback-ref": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/use-callback-ref/-/use-callback-ref-1.3.3.tgz", + "integrity": "sha512-jQL3lRnocaFtu3V00JToYz/4QkNWswxijDaCVNZRiRTO3HQDLsdu1ZtmIUvV4yPp+rvWm5j0y0TG/S61cuijTg==", + "dev": true, + "license": "MIT", + "dependencies": { + "tslib": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/use-composed-ref": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/use-composed-ref/-/use-composed-ref-1.4.0.tgz", + "integrity": "sha512-djviaxuOOh7wkj0paeO1Q/4wMZ8Zrnag5H6yBvzN7AKKe8beOaED9SF5/ByLqsku8NP4zQqsvM2u3ew/tJK8/w==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/use-isomorphic-layout-effect": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/use-isomorphic-layout-effect/-/use-isomorphic-layout-effect-1.2.1.tgz", + "integrity": "sha512-tpZZ+EX0gaghDAiFR37hj5MgY6ZN55kLiPkJsKxBMZ6GZdOSPJXiOzPM984oPYZ5AnehYx5WQp1+ME8I/P/pRA==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/use-latest": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/use-latest/-/use-latest-1.3.0.tgz", + "integrity": "sha512-mhg3xdm9NaM8q+gLT8KryJPnRFOz1/5XPBhmDEVZK1webPzDjrPk7f/mbpeLqTgB9msytYWANxgALOCJKnLvcQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "use-isomorphic-layout-effect": "^1.1.1" + }, + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/use-sidecar": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/use-sidecar/-/use-sidecar-1.1.3.tgz", + "integrity": "sha512-Fedw0aZvkhynoPYlA5WXrMCAMm+nSWdZt6lzJQ7Ok8S6Q+VsHmHpRWndVRJ8Be0ZbkfPc5LRYH+5XrzXcEeLRQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "detect-node-es": "^1.1.0", + "tslib": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/use-sync-external-store": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/use-sync-external-store/-/use-sync-external-store-1.7.0.tgz", + "integrity": "sha512-6L+EeigHMQhdaIPNIFUKwfWJSwWFQ8gJbJ2DLOs5sDIegTwR9fRxvnM3uciHKjIZhFz+KAv2emhWMRvDmMcY8A==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, + "node_modules/whatwg-mimetype": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-3.0.0.tgz", + "integrity": "sha512-nt+N2dzIutVRxARx1nghPKGv1xHikU7HKdfafKkLNLindmPU/ch3U31NOCGGA/dmPcmb1VlofO0vnKAcsm0o/Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/ws": { + "version": "8.21.3", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.3.tgz", + "integrity": "sha512-201TZ/kPWxoPr/OKWjquZR1SWKXcvxdH+e1xrx89b3YbmzLMFCLfnaG1HFIgWzJOEWZ7MvpK++odZufgYR50Rw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/zod": { + "version": "4.6.5", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.6.5.tgz", + "integrity": "sha512-v5l/aFXZQeai4awLbOpSoHecE9UiMrnfx75tEXLjNonXVARxQ5mOeipTjROUchszUNCqnE+hqAMujRsRHsut2Q==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/zustand": { + "version": "5.0.15", + "resolved": "https://registry.npmjs.org/zustand/-/zustand-5.0.15.tgz", + "integrity": "sha512-MpSEjRiBkA9crSYeOUH32rJC7SVqAbm0Fqcqge/bUi2PPoLcBWKOsG+C8mevmpr8TwXHBVkChbbJiyvkE+i/3A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.20.0" + }, + "peerDependencies": { + "@types/react": ">=18.0.0", + "immer": ">=9.0.6", + "react": ">=18.0.0", + "use-sync-external-store": ">=1.2.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "immer": { + "optional": true + }, + "react": { + "optional": true + }, + "use-sync-external-store": { + "optional": true + } + } + } + } +} diff --git a/chat-sdk/package.json b/chat-sdk/package.json new file mode 100644 index 0000000000..b13f75656a --- /dev/null +++ b/chat-sdk/package.json @@ -0,0 +1,108 @@ +{ + "name": "windmill-chat", + "description": "Build chat interfaces on Windmill flows deployed in chat mode, from any frontend or raw app", + "version": "1.810.0", + "author": "Ruben Fiszel", + "license": "Apache-2.0", + "homepage": "https://github.com/windmill-labs/windmill/tree/main/chat-sdk#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/windmill-labs/windmill.git", + "directory": "chat-sdk" + }, + "bugs": { + "url": "https://github.com/windmill-labs/windmill/issues" + }, + "keywords": [ + "windmill", + "chat", + "ai", + "agent", + "react" + ], + "sideEffects": false, + "type": "module", + "main": "dist/index.cjs", + "module": "dist/index.js", + "types": "dist/index.d.ts", + "exports": { + ".": { + "import": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + }, + "require": { + "types": "./dist/index.d.ts", + "default": "./dist/index.cjs" + } + }, + "./react": { + "import": { + "types": "./dist/react.d.ts", + "default": "./dist/react.js" + }, + "require": { + "types": "./dist/react.d.ts", + "default": "./dist/react.cjs" + } + }, + "./ai-sdk": { + "import": { + "types": "./dist/ai-sdk.d.ts", + "default": "./dist/ai-sdk.js" + }, + "require": { + "types": "./dist/ai-sdk.d.ts", + "default": "./dist/ai-sdk.cjs" + } + }, + "./assistant-ui": { + "import": { + "types": "./dist/assistant-ui.d.ts", + "default": "./dist/assistant-ui.js" + }, + "require": { + "types": "./dist/assistant-ui.d.ts", + "default": "./dist/assistant-ui.cjs" + } + } + }, + "files": [ + "dist", + "README.md" + ], + "scripts": { + "build": "tsdown && tsc -p tsconfig.build.json", + "check": "tsc --noEmit", + "test": "bun test" + }, + "peerDependencies": { + "@assistant-ui/react": ">=0.15", + "ai": ">=5", + "react": ">=18" + }, + "peerDependenciesMeta": { + "@assistant-ui/react": { + "optional": true + }, + "ai": { + "optional": true + }, + "react": { + "optional": true + } + }, + "devDependencies": { + "@ai-sdk/react": "^4.0.102", + "@assistant-ui/react": "^0.15.19", + "@happy-dom/global-registrator": "^20.14.5", + "@types/bun": "^1.3.0", + "@types/react": "^19.0.0", + "@types/react-dom": "^19.3.0", + "ai": "^7.0.99", + "react": "^19.0.0", + "react-dom": "^19.3.0", + "tsdown": "^0.12.9", + "typescript": "^5.4.5" + } +} diff --git a/chat-sdk/src/ai-sdk.ts b/chat-sdk/src/ai-sdk.ts new file mode 100644 index 0000000000..ba500446a3 --- /dev/null +++ b/chat-sdk/src/ai-sdk.ts @@ -0,0 +1,324 @@ +import type { ChatTransport, UIMessage, UIMessageChunk, UIMessagePart } from 'ai' +import { WindmillApiError, WindmillChatApi, type WindmillChatApiOptions } from './api' +import { followJob } from './follow' +import type { AgentStreamEvent } from './stream' +import type { ChatMessage, Conversation } from './types' +import { + conversationIdFor, + errorResultMessage, + extractChatAnswer, + isAbortError, + isErrorResult, + parseJsonOr, + randomId +} from './utils' + +export interface WindmillChatTransportOptions extends WindmillChatApiOptions { + /** Path of a deployed flow with chat mode enabled, e.g. `f/support/assistant`. */ + flowPath: string + /** Extra flow inputs sent with every message; `sendMessage(msg, { body })` adds per-message ones. */ + inputs?: Record +} + +export interface WindmillChatTransport + extends ChatTransport { + /** The Windmill conversation id behind an AI SDK chat id (a UUID chat id is used as is). */ + conversationId(chatId: string): string + /** Server history of a chat as `UIMessage`s, oldest first, for `useChat({ messages })`. Needs `flow_conversations:read`. */ + loadMessages(chatId: string, options?: { page?: number; perPage?: number }): Promise + /** The user's conversations for this flow, most recent first. */ + listConversations(options?: { page?: number; perPage?: number }): Promise + deleteConversation(chatId: string): Promise +} + +interface JobEntry { + jobId: string + offset?: number + done: boolean +} + +/** + * A Vercel AI SDK `ChatTransport` over a chat-mode flow: `useChat({ transport })` + * (and AI Elements, which builds on it) then work against Windmill unchanged. + * The chat id is the conversation, so a UUID id lines up with server history. + */ +export function createWindmillChatTransport( + options: WindmillChatTransportOptions +): WindmillChatTransport { + const api = new WindmillChatApi(options) + // One in-flight or finished job per chat, for `reconnectToStream`. + const jobs = new Map() + + return { + conversationId: conversationIdFor, + + async sendMessages({ chatId, messages, abortSignal, body }) { + const last = messages[messages.length - 1] + if (!last || last.role !== 'user') { + throw new Error('windmill-chat: the last message must be a user message') + } + if (last.parts.some((p) => p.type === 'file')) { + throw new Error( + 'windmill-chat: attachments are not supported; upload the file yourself and pass its reference through `body`' + ) + } + const text = last.parts + .filter((p): p is Extract, { type: 'text' }> => p.type === 'text') + .map((p) => p.text) + .join('\n') + const memoryId = conversationIdFor(chatId) + const jobId = await api.runFlow( + options.flowPath, + { ...options.inputs, ...(body as Record | undefined), user_message: text }, + { memoryId, signal: abortSignal } + ) + const entry: JobEntry = { jobId, done: false } + jobs.set(chatId, entry) + return chunkStream(api, entry, abortSignal) + }, + + async reconnectToStream({ chatId, abortSignal }) { + const entry = jobs.get(chatId) + if (!entry || entry.done) return null + return chunkStream(api, entry, abortSignal) + }, + + async loadMessages(chatId, pagination) { + // A chat that hasn't sent anything yet has no conversation on the server. + const rows = await api.listMessages(conversationIdFor(chatId), pagination).catch((e) => { + if (e instanceof WindmillApiError && e.status === 404) return [] + throw e + }) + return toUIMessages( + rows.map((row) => ({ + id: row.id, + serverId: row.id, + role: row.message_type, + content: row.content, + success: row.success ?? true, + createdAt: row.created_at, + jobId: row.job_id ?? undefined, + stepName: row.step_name ?? undefined, + pending: false, + seq: row.created_seq, + tool: toolFromRowContent(row.message_type, row.content, row.success ?? true) + })) + ) as UI_MESSAGE[] + }, + + async listConversations(pagination) { + const rows = await api.listConversations(options.flowPath, pagination) + return rows.map((row) => ({ + id: row.id, + title: row.title ?? undefined, + createdAt: row.created_at, + updatedAt: row.updated_at + })) + }, + + async deleteConversation(chatId) { + await api.deleteConversation(conversationIdFor(chatId)) + jobs.delete(chatId) + } + } +} + +function toolFromRowContent(role: string, content: string, success: boolean): ChatMessage['tool'] { + if (role !== 'tool') return undefined + const name = /^Used (.+) tool$/.exec(content)?.[1] ?? /^Error executing (.+)$/.exec(content)?.[1] + return name ? { name, status: success ? 'success' : 'error' } : undefined +} + +/** Streams a job's answer as AI SDK chunks; resumes from `entry.offset` when the job is already running. */ +function chunkStream( + api: WindmillChatApi, + entry: JobEntry, + signal: AbortSignal | undefined +): ReadableStream { + return new ReadableStream({ + async start(controller) { + const parts = new PartWriter((chunk) => controller.enqueue(chunk)) + parts.emit({ type: 'start' }) + try { + let failure: string | undefined + for await (const event of followJob(api, entry.jobId, { + signal, + streamOffset: entry.offset, + onOffset: (offset) => { + entry.offset = offset + } + })) { + if (event.type === 'stream') { + for (const e of event.events) parts.apply(e) + continue + } + parts.closeOpen() + failure = await failureText(api, entry.jobId, event.result, signal) + if (failure === undefined && !parts.streamedText) { + // No agent streamed: the flow's result is the answer. + const answer = extractChatAnswer(event.result) + if (answer !== undefined) { + parts.text(answer) + parts.closeOpen() + } + } + } + entry.done = true + parts.emit(failure === undefined ? { type: 'finish' } : { type: 'error', errorText: failure }) + } catch (e) { + if (!isAbortError(e)) { + parts.closeOpen() + parts.emit({ type: 'error', errorText: e instanceof Error ? e.message : String(e) }) + } + } finally { + controller.close() + } + } + }) +} + +/** A completed flow's error, when the job did fail (the envelope alone is a legitimate result). */ +async function failureText( + api: WindmillChatApi, + jobId: string, + result: unknown, + signal: AbortSignal | undefined +): Promise { + if (!isErrorResult(result)) return undefined + const failed = await api + .getCompletedResult(jobId, signal) + .then((r) => r.success === false) + .catch(() => true) + return failed ? errorResultMessage(result) : undefined +} + +/** + * Turns agent events into AI SDK chunks. Text and reasoning are open parts that a + * tool call closes (a new round starts new parts); tool calls are `dynamic-tool` + * parts, since the UI declares no tools of its own. + */ +class PartWriter { + streamedText = false + #textId: string | undefined + #reasoningId: string | undefined + #started = new Set() + #inputSent = new Set() + + constructor(readonly emit: (chunk: UIMessageChunk) => void) {} + + text(delta: string): void { + this.streamedText = true + if (!this.#textId) { + this.#textId = randomId() + this.emit({ type: 'text-start', id: this.#textId }) + } + this.emit({ type: 'text-delta', id: this.#textId, delta }) + } + + reasoning(delta: string): void { + if (!this.#reasoningId) { + this.#reasoningId = randomId() + this.emit({ type: 'reasoning-start', id: this.#reasoningId }) + } + this.emit({ type: 'reasoning-delta', id: this.#reasoningId, delta }) + } + + closeOpen(): void { + if (this.#reasoningId) { + this.emit({ type: 'reasoning-end', id: this.#reasoningId }) + this.#reasoningId = undefined + } + if (this.#textId) { + this.emit({ type: 'text-end', id: this.#textId }) + this.#textId = undefined + } + } + + apply(event: AgentStreamEvent): void { + switch (event.type) { + case 'token_delta': + this.text(event.content) + break + case 'reasoning_token_delta': + this.reasoning(event.content) + break + case 'tool_call': + this.closeOpen() + this.#toolStart(event.call_id, event.function_name) + break + case 'tool_call_arguments': + this.closeOpen() + this.#toolStart(event.call_id, event.function_name) + this.#inputSent.add(event.call_id) + this.emit({ + type: 'tool-input-available', + toolCallId: event.call_id, + toolName: event.function_name, + input: parseJsonOr(event.arguments), + dynamic: true + }) + break + case 'tool_execution': + this.closeOpen() + this.#toolStart(event.call_id, event.function_name) + break + case 'tool_result': + this.#toolStart(event.call_id, event.function_name) + if (!this.#inputSent.has(event.call_id)) { + this.#inputSent.add(event.call_id) + this.emit({ + type: 'tool-input-available', + toolCallId: event.call_id, + toolName: event.function_name, + input: undefined, + dynamic: true + }) + } + this.emit( + event.success + ? { type: 'tool-output-available', toolCallId: event.call_id, output: parseJsonOr(event.result), dynamic: true } + : { type: 'tool-output-error', toolCallId: event.call_id, errorText: event.result, dynamic: true } + ) + break + } + } + + #toolStart(callId: string, name: string): void { + if (this.#started.has(callId)) return + this.#started.add(callId) + this.emit({ type: 'tool-input-start', toolCallId: callId, toolName: name, dynamic: true }) + } +} + +/** + * `ChatMessage`s (Windmill's role-per-row model) as `UIMessage`s: an assistant + * turn becomes one message whose parts carry its text, reasoning and tool calls. + */ +export function toUIMessages(messages: ChatMessage[]): UIMessage[] { + const out: UIMessage[] = [] + for (const m of messages) { + if (m.role === 'user' || m.role === 'system') { + out.push({ id: m.id, role: m.role, parts: [{ type: 'text', text: m.content }] }) + continue + } + let target = out[out.length - 1] + if (!target || target.role !== 'assistant') { + target = { id: m.id, role: 'assistant', parts: [] } + out.push(target) + } + if (m.role === 'tool') { + const toolCallId = m.tool?.callId ?? m.id + const toolName = m.tool?.name ?? 'tool' + const input = parseJsonOr(m.tool?.arguments) + target.parts.push( + m.success + ? { type: 'dynamic-tool', toolName, toolCallId, state: 'output-available', input, output: parseJsonOr(m.tool?.result) ?? m.content } + : { type: 'dynamic-tool', toolName, toolCallId, state: 'output-error', input, errorText: m.tool?.result ?? m.content } + ) + continue + } + if (m.reasoning) target.parts.push({ type: 'reasoning', text: m.reasoning, state: 'done' }) + if (m.content) target.parts.push({ type: 'text', text: m.content, state: 'done' }) + } + return out +} diff --git a/chat-sdk/src/api.ts b/chat-sdk/src/api.ts new file mode 100644 index 0000000000..0570154ec2 --- /dev/null +++ b/chat-sdk/src/api.ts @@ -0,0 +1,294 @@ +import type { FetchLike, TokenSource } from './types' + +export interface WindmillChatApiOptions { + baseUrl: string + workspace: string + /** Omit to rely on the session cookie of the Windmill origin. */ + token?: TokenSource + fetch?: FetchLike +} + +export class WindmillApiError extends Error { + constructor( + message: string, + readonly status: number + ) { + super(message) + this.name = 'WindmillApiError' + } +} + +export interface FlowConversation { + id: string + workspace_id: string + flow_path: string + title?: string | null + created_at: string + updated_at: string + created_by: string +} + +export interface FlowConversationMessage { + id: string + conversation_id: string + message_type: 'user' | 'assistant' | 'system' | 'tool' + content: string + job_id?: string | null + created_at: string + created_seq: number + step_name?: string | null + success?: boolean +} + +export type JobUpdateEvent = + | { + type: 'update' + running?: boolean + completed?: boolean + new_result_stream?: string + stream_offset?: number + only_result?: unknown + flow_stream_job_id?: string + } + | { type: 'error'; error: string } + | { type: 'notfound' } + | { type: 'timeout' } + | { type: 'ping' } + +export interface CompletedJobResult { + completed: boolean + success?: boolean + result?: unknown +} + +/** The part of a flow job's status that names the jobs its steps ran as. */ +export interface FlowJobStatus { + flow_status?: { + modules?: FlowStepStatus[] | null + failure_module?: FlowStepStatus | null + preprocessor_module?: FlowStepStatus | null + } | null +} + +export interface FlowStepStatus { + job?: string | null + flow_jobs?: string[] | null +} + +/** Thin client over the Windmill endpoints a chat-mode flow uses. */ +export class WindmillChatApi { + readonly #baseUrl: string + readonly #workspace: string + readonly #token: TokenSource | undefined + readonly #fetch: FetchLike + + constructor(options: WindmillChatApiOptions) { + this.#baseUrl = normalizeBaseUrl(options.baseUrl) + this.#workspace = options.workspace + this.#token = options.token + this.#fetch = options.fetch ?? ((input, init) => globalThis.fetch(input, init)) + } + + /** Starts a turn: runs the flow with `memory_id` set to the conversation id. Returns the job id. */ + async runFlow( + flowPath: string, + args: Record, + options: { memoryId: string; signal?: AbortSignal } + ): Promise { + const res = await this.#request(`jobs/run/f/${encodePath(flowPath)}`, { + method: 'POST', + query: { memory_id: options.memoryId, skip_preprocessor: 'true' }, + body: args, + signal: options.signal + }) + return (await res.text()).trim() + } + + /** + * One server-sent-events connection to a job's updates. The server closes it after + * `TIMEOUT_SSE_STREAM` (a `timeout` event); resume by calling again with the last + * `stream_offset`, never by re-running the flow. + */ + async *streamJob( + jobId: string, + options: { streamOffset?: number; signal?: AbortSignal } = {} + ): AsyncGenerator { + const query: Record = { fast: 'true', only_result: 'true' } + if (options.streamOffset !== undefined) { + query.stream_offset = String(options.streamOffset) + } + const res = await this.#request(`jobs_u/getupdate_sse/${encodeURIComponent(jobId)}`, { + query, + accept: 'text/event-stream', + signal: options.signal + }) + if (!res.body) { + throw new WindmillApiError('The job update stream has no body', res.status) + } + for await (const data of readServerSentEvents(res.body)) { + try { + yield JSON.parse(data) as JobUpdateEvent + } catch { + // A frame that isn't JSON carries nothing the chat can use. + } + } + } + + async getCompletedResult(jobId: string, signal?: AbortSignal): Promise { + const res = await this.#request( + `jobs_u/completed/get_result_maybe/${encodeURIComponent(jobId)}`, + { signal } + ) + return (await res.json()) as CompletedJobResult + } + + /** A flow job with its status: the step job ids are what persisted messages carry as `job_id`. */ + async getFlowJob(jobId: string, signal?: AbortSignal): Promise { + const res = await this.#request(`jobs_u/get/${encodeURIComponent(jobId)}`, { + query: { no_logs: 'true' }, + signal + }) + return (await res.json()) as FlowJobStatus + } + + async cancelJob(jobId: string, reason = 'Stopped from the chat'): Promise { + await this.#request(`jobs_u/queue/cancel/${encodeURIComponent(jobId)}`, { + method: 'POST', + body: { reason } + }) + } + + async listConversations( + flowPath: string, + options: { page?: number; perPage?: number; signal?: AbortSignal } = {} + ): Promise { + const res = await this.#request('flow_conversations/list', { + query: pagination(options, { flow_path: flowPath }), + signal: options.signal + }) + return (await res.json()) as FlowConversation[] + } + + /** + * Without `afterSeq`: one page counted from the newest message, returned oldest first. + * With `afterSeq`: the messages created after that cursor, oldest first. + */ + async listMessages( + conversationId: string, + options: { page?: number; perPage?: number; afterSeq?: number; signal?: AbortSignal } = {} + ): Promise { + const extra: Record = {} + if (options.afterSeq !== undefined) extra.after_seq = String(options.afterSeq) + const res = await this.#request( + `flow_conversations/${encodeURIComponent(conversationId)}/messages`, + { query: pagination(options, extra), signal: options.signal } + ) + return (await res.json()) as FlowConversationMessage[] + } + + async deleteConversation(conversationId: string): Promise { + await this.#request(`flow_conversations/delete/${encodeURIComponent(conversationId)}`, { + method: 'DELETE' + }) + } + + async #request( + path: string, + init: { + method?: string + query?: Record + body?: unknown + accept?: string + signal?: AbortSignal + } = {} + ): Promise { + const url = new URL(`${this.#baseUrl}/api/w/${encodeURIComponent(this.#workspace)}/${path}`) + for (const [k, v] of Object.entries(init.query ?? {})) url.searchParams.set(k, v) + + const headers: Record = {} + if (init.accept) headers['Accept'] = init.accept + if (init.body !== undefined) headers['Content-Type'] = 'application/json' + const token = typeof this.#token === 'function' ? await this.#token() : this.#token + if (token) headers['Authorization'] = `Bearer ${token}` + + const res = await this.#fetch(url.toString(), { + method: init.method ?? 'GET', + headers, + body: init.body === undefined ? undefined : JSON.stringify(init.body), + // A token must not be paired with ambient cookies; without one, the cookie is + // the credential and only rides same-origin requests. + credentials: token ? 'omit' : 'same-origin', + signal: init.signal + }) + if (!res.ok) { + const text = await res.text().catch(() => '') + throw new WindmillApiError( + `${init.method ?? 'GET'} ${path} failed (${res.status})${text ? `: ${text}` : ''}`, + res.status + ) + } + return res + } +} + +export function normalizeBaseUrl(baseUrl: string): string { + return baseUrl.replace(/\/+$/, '').replace(/\/api$/, '') +} + +function encodePath(path: string): string { + return path.split('/').map(encodeURIComponent).join('/') +} + +function pagination( + options: { page?: number; perPage?: number }, + extra: Record +): Record { + const query = { ...extra } + if (options.page !== undefined) query.page = String(options.page) + if (options.perPage !== undefined) query.per_page = String(options.perPage) + return query +} + +/** Yields the `data` payload of each event in a `text/event-stream` body. */ +export async function* readServerSentEvents( + body: ReadableStream +): AsyncGenerator { + const reader = body.getReader() + const decoder = new TextDecoder() + let buffer = '' + // A CR ending a chunk may be half of a CRLF; it waits for the next chunk. + let carry = '' + try { + while (true) { + const { value, done } = await reader.read() + if (done) break + let text = carry + decoder.decode(value, { stream: true }) + carry = '' + if (text.endsWith('\r')) { + carry = '\r' + text = text.slice(0, -1) + } + buffer += text.replace(/\r\n?/g, '\n') + let end: number + while ((end = buffer.indexOf('\n\n')) !== -1) { + const data = eventData(buffer.slice(0, end)) + buffer = buffer.slice(end + 2) + if (data !== undefined) yield data + } + } + if (carry) buffer += '\n' + const data = eventData(buffer) + if (data !== undefined) yield data + } finally { + // Closes the connection when the consumer stops early. + reader.cancel().catch(() => {}) + } +} + +function eventData(block: string): string | undefined { + const lines = block + .split('\n') + .filter((line) => line.startsWith('data:')) + .map((line) => line.slice(line.startsWith('data: ') ? 6 : 5)) + return lines.length > 0 ? lines.join('\n') : undefined +} diff --git a/chat-sdk/src/assistant-ui.ts b/chat-sdk/src/assistant-ui.ts new file mode 100644 index 0000000000..ef22f28531 --- /dev/null +++ b/chat-sdk/src/assistant-ui.ts @@ -0,0 +1,124 @@ +import { + useExternalStoreRuntime, + type AppendMessage, + type AssistantRuntime, + type ExternalStoreAdapter, + type ThreadMessageLike +} from '@assistant-ui/react' +import { useEffect, useMemo } from 'react' +import { useWindmillChat } from './react' +import type { ChatMessage, ChatOptions } from './types' +import { parseJsonOr } from './utils' + +/** One assistant turn: the assistant and tool rows between two user messages. */ +export interface WindmillTurn { + id: string + role: 'user' | 'assistant' | 'system' + messages: ChatMessage[] +} + +export interface WindmillRuntimeOptions extends ChatOptions { + /** Load the conversation list on mount so `ThreadListPrimitive` has something to show. Default true. */ + threadList?: boolean +} + +/** + * An assistant-ui runtime over a chat-mode flow, for `AssistantRuntimeProvider`. + * Conversations become threads, so the thread list primitives switch, create and + * delete Windmill conversations. + */ +export function useWindmillRuntime(options: WindmillRuntimeOptions): AssistantRuntime { + const chat = useWindmillChat(options) + const turns = useMemo(() => groupTurns(chat.messages), [chat.messages]) + const withThreadList = options.threadList !== false + useEffect(() => { + if (withThreadList) chat.loadConversations().catch(() => {}) + }, [chat.chat, withThreadList]) + + const adapter: ExternalStoreAdapter = { + messages: turns, + isRunning: chat.status === 'submitted' || chat.status === 'streaming', + convertMessage: toThreadMessage, + onNew: async (message: AppendMessage) => { + if (message.role !== 'user') return + await chat.sendMessage(appendedText(message)) + }, + onCancel: async () => { + await chat.stop() + }, + adapters: withThreadList + ? { + threadList: { + threadId: chat.conversationId, + threads: chat.conversations.map((c) => ({ status: 'regular' as const, id: c.id, title: c.title })), + onSwitchToNewThread: () => chat.newConversation(), + onSwitchToThread: (id) => chat.selectConversation(id), + onDelete: (id) => chat.deleteConversation(id) + } + } + : undefined + } + return useExternalStoreRuntime(adapter) +} + +function appendedText(message: AppendMessage): string { + return message.content + .filter((p): p is { type: 'text'; text: string } => p.type === 'text') + .map((p) => p.text) + .join('\n') +} + +/** Folds the role-per-row message list into turns: one entry per user message, one per answer. */ +export function groupTurns(messages: ChatMessage[]): WindmillTurn[] { + const turns: WindmillTurn[] = [] + for (const m of messages) { + const last = turns[turns.length - 1] + if (m.role === 'user' || m.role === 'system' || !last || last.role !== 'assistant') { + turns.push({ id: m.id, role: m.role === 'tool' ? 'assistant' : m.role, messages: [m] }) + } else { + last.messages.push(m) + } + } + return turns +} + +/** A turn as assistant-ui content parts; the status reflects streaming and a failed flow. */ +export function toThreadMessage(turn: WindmillTurn): ThreadMessageLike { + const first = turn.messages[0] + const createdAt = new Date(first.createdAt) + if (turn.role !== 'assistant') { + return { id: turn.id, role: turn.role, createdAt, content: [{ type: 'text', text: first.content }] } + } + const content: ThreadContentPart[] = [] + for (const m of turn.messages) { + if (m.role === 'tool') { + const args = parseJsonOr(m.tool?.arguments) + content.push({ + type: 'tool-call', + toolCallId: m.tool?.callId ?? m.id, + toolName: m.tool?.name ?? 'tool', + args: (isJsonObject(args) ? args : args === undefined ? {} : { input: args }) as ToolCallArgs, + argsText: m.tool?.arguments ?? '', + result: m.tool?.status === 'running' ? undefined : (parseJsonOr(m.tool?.result) ?? m.content), + isError: m.tool?.status === 'error' + }) + continue + } + if (m.reasoning) content.push({ type: 'reasoning', text: m.reasoning }) + if (m.content) content.push({ type: 'text', text: m.content }) + } + const last = turn.messages[turn.messages.length - 1] + const status: ThreadMessageLike['status'] = turn.messages.some((m) => m.pending) + ? { type: 'running' } + : last.role === 'assistant' && !last.success + ? { type: 'incomplete', reason: 'error', error: last.content } + : { type: 'complete', reason: 'stop' } + return { id: turn.id, role: 'assistant', createdAt, content, status } +} + +type ThreadContentPart = Exclude[number] +type ToolCallArgs = Extract['args'] + +function isJsonObject(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} diff --git a/chat-sdk/src/chat.ts b/chat-sdk/src/chat.ts new file mode 100644 index 0000000000..866d185d84 --- /dev/null +++ b/chat-sdk/src/chat.ts @@ -0,0 +1,744 @@ +import { + WindmillApiError, + WindmillChatApi, + type FlowConversation, + type FlowConversationMessage +} from './api' +import { resolveConfig, type ResolvedConfig } from './config' +import { followJob } from './follow' +import { createLocalHistory, type LocalHistory } from './history' +import type { AgentStreamEvent } from './stream' +import type { + Chat, + ChatMessage, + ChatOptions, + ChatState, + Conversation, + ToolInvocation +} from './types' +import { + conversationTitle, + errorResultMessage, + extractChatAnswer, + isAbortError, + isErrorResult, + now, + randomId, + sleep +} from './utils' + +const POLL_INTERVAL_MS = 1000 +/** Local history mirrors the state; a stream of deltas is coalesced into one write. */ +const PERSIST_DEBOUNCE_MS = 250 +/** Messages persist from spawned tasks that can land just after the flow completes. */ +const RECONCILE_ATTEMPTS = 3 +const RECONCILE_DELAY_MS = 400 + +interface Turn { + controller: AbortController + conversationId: string + /** Id of the turn's user message; the answer is whatever follows it. */ + userMessageId: string + jobId?: string + /** The flow job and its step jobs; a persisted answer carries one of them as `job_id`. */ + jobIds?: Set + /** Id of the streaming assistant message; cleared when a tool call ends the round. */ + assistantId?: string + streamedText: boolean +} + +export function createChat(options: ChatOptions): Chat { + return new ChatImpl(options) +} + +class ChatImpl implements Chat { + readonly #config: ResolvedConfig + readonly #api: WindmillChatApi + readonly #local: LocalHistory + readonly #listeners = new Set<(state: ChatState) => void>() + #state: ChatState + #turn: Turn | undefined + #page = 1 + #persistTimer: ReturnType | undefined + + constructor(options: ChatOptions) { + this.#config = resolveConfig(options) + this.#api = new WindmillChatApi({ + baseUrl: this.#config.baseUrl, + workspace: this.#config.workspace, + token: this.#config.token, + fetch: this.#config.fetch + }) + this.#local = createLocalHistory( + this.#config.storage, + `windmill-chat:${this.#config.baseUrl}:${this.#config.workspace}:${this.#config.flowPath}` + + (this.#config.storageKey ? `:${this.#config.storageKey}` : '') + ) + this.#state = { + conversationId: undefined, + messages: [], + status: 'idle', + error: undefined, + conversations: [], + history: this.#config.history, + loadingMessages: false, + hasMoreMessages: false + } + } + + getState = (): ChatState => this.#state + + subscribe = (listener: (state: ChatState) => void): (() => void) => { + this.#listeners.add(listener) + listener(this.#state) + return () => { + this.#listeners.delete(listener) + } + } + + sendMessage = async ( + text: string, + options: { inputs?: Record } = {} + ): Promise => { + const content = text.trim() + if (!content) return + if (this.#turn) { + throw new Error('windmill-chat: a message is already being answered; call stop() first') + } + const isNew = this.#state.conversationId === undefined + const conversationId = this.#state.conversationId ?? randomId() + const turn: Turn = { + controller: new AbortController(), + conversationId, + userMessageId: `pending-${randomId()}`, + streamedText: false + } + this.#turn = turn + + const timestamp = now() + const conversation: Conversation = this.#state.conversations.find( + (c) => c.id === conversationId + ) ?? { id: conversationId, title: conversationTitle(content), createdAt: timestamp, updatedAt: timestamp } + const touched = { ...conversation, updatedAt: timestamp } + this.#set({ + conversationId, + conversations: [touched, ...this.#state.conversations.filter((c) => c.id !== conversationId)], + messages: [ + ...this.#state.messages, + { id: turn.userMessageId, role: 'user', content, success: true, createdAt: timestamp, pending: true } + ], + status: 'submitted', + error: undefined + }) + this.#rememberConversation() + + try { + turn.jobId = await this.#api.runFlow( + this.#config.flowPath, + { ...this.#config.inputs, ...options.inputs, user_message: content }, + { memoryId: conversationId, signal: turn.controller.signal } + ) + const stopPolling = this.#state.history === 'server' ? this.#startPolling(turn) : () => {} + let result: unknown + try { + result = await this.#follow(turn, stopPolling) + } finally { + stopPolling() + } + await this.#finishTurn(turn, result, isNew) + } catch (e) { + // stop() and a conversation switch abort the turn and settle the state themselves. + if (turn.controller.signal.aborted || isAbortError(e)) return + this.#failTurn(turn, e) + } finally { + if (this.#turn === turn) this.#turn = undefined + } + } + + stop = async (): Promise => { + const turn = this.#turn + if (!turn) return + this.#detachTurn() + if (this.#state.conversationId === turn.conversationId) { + this.#set({ messages: finalized(this.#state.messages), status: 'idle' }) + this.#persistLocal() + } + if (turn.jobId) { + // Needs `jobs:write` on a token; the stream is closed either way. + await this.#api.cancelJob(turn.jobId).catch(() => {}) + } + if (this.#state.history !== 'server') return + // A cancelled flow persists its failure as the assistant's answer. Picked up only + // while the conversation is still idle: a turn started meanwhile owns the state. + await sleep(RECONCILE_DELAY_MS).catch(() => {}) + if (this.#turn || this.#state.conversationId !== turn.conversationId) return + await this.#syncFromServer(turn.conversationId).catch(() => {}) + } + + newConversation = (): void => { + this.#leaveConversation() + this.#page = 1 + this.#set({ + conversationId: undefined, + messages: [], + status: 'idle', + error: undefined, + loadingMessages: false, + hasMoreMessages: false + }) + } + + selectConversation = async (conversationId: string): Promise => { + if (conversationId === this.#state.conversationId) return + this.#leaveConversation() + this.#page = 1 + this.#set({ + conversationId, + messages: [], + status: 'idle', + error: undefined, + loadingMessages: true, + hasMoreMessages: false + }) + if (this.#state.history !== 'server') { + this.#set({ + messages: this.#state.history === 'local' ? this.#local.getMessages(conversationId) : [], + loadingMessages: false + }) + return + } + try { + const rows = await this.#api.listMessages(conversationId, { + perPage: this.#config.pageSize + }) + if (this.#state.conversationId !== conversationId) return + this.#set({ + messages: rows.map(fromRow), + loadingMessages: false, + hasMoreMessages: rows.length === this.#config.pageSize + }) + } catch (e) { + if (this.#state.conversationId !== conversationId) return + if (this.#fallBackToLocal(e)) { + this.#set({ messages: this.#local.getMessages(conversationId), loadingMessages: false }) + return + } + this.#set({ loadingMessages: false, status: 'error', error: toError(e) }) + } + } + + loadConversations = async ( + options: { page?: number; perPage?: number } = {} + ): Promise => { + const page = options.page ?? 1 + let conversations: Conversation[] + if (this.#state.history === 'server') { + try { + const rows = await this.#api.listConversations(this.#config.flowPath, { + page, + perPage: options.perPage ?? this.#config.pageSize + }) + conversations = rows.map(fromConversation) + } catch (e) { + if (!this.#fallBackToLocal(e)) throw e + conversations = this.#local.listConversations() + } + } else { + conversations = this.#state.history === 'local' ? this.#local.listConversations() : [] + } + const known = new Set(this.#state.conversations.map((c) => c.id)) + this.#set({ + conversations: + page === 1 + ? conversations + : [...this.#state.conversations, ...conversations.filter((c) => !known.has(c.id))] + }) + return conversations + } + + deleteConversation = async (conversationId: string): Promise => { + if (this.#state.conversationId === conversationId) { + // Nothing of the current turn may be written back under the deleted id. + this.#detachTurn() + clearTimeout(this.#persistTimer) + this.#persistTimer = undefined + this.newConversation() + } + if (this.#state.history === 'server') { + await this.#api.deleteConversation(conversationId) + } else if (this.#state.history === 'local') { + this.#local.deleteConversation(conversationId) + } + this.#set({ conversations: this.#state.conversations.filter((c) => c.id !== conversationId) }) + } + + loadOlderMessages = async (): Promise => { + const conversationId = this.#state.conversationId + if ( + !conversationId || + this.#state.history !== 'server' || + !this.#state.hasMoreMessages || + this.#state.loadingMessages + ) { + return + } + const page = this.#page + 1 + this.#set({ loadingMessages: true }) + try { + const rows = await this.#api.listMessages(conversationId, { + page, + perPage: this.#config.pageSize + }) + if (this.#state.conversationId !== conversationId) return + const known = new Set(this.#state.messages.map((m) => m.serverId ?? m.id)) + this.#page = page + this.#set({ + messages: [...rows.map(fromRow).filter((m) => !known.has(m.id)), ...this.#state.messages], + hasMoreMessages: rows.length === this.#config.pageSize + }) + } finally { + if (this.#state.conversationId === conversationId) this.#set({ loadingMessages: false }) + } + } + + destroy = (): void => { + this.#leaveConversation() + } + + // ---- turn internals ---- + + async #follow(turn: Turn, onStreamStart: () => void): Promise { + let started = false + for await (const event of followJob(this.#api, turn.jobId!, { signal: turn.controller.signal })) { + if (event.type === 'completed') return event.result + if (!started) { + started = true + // Persisted rows for the streaming step would duplicate what is streaming. + onStreamStart() + } + this.#applyEvents(turn, event.events) + } + throw new Error('windmill-chat: the job stream ended before the flow completed') + } + + #applyEvents(turn: Turn, events: AgentStreamEvent[]): void { + if (events.length === 0 || !this.#turnActive(turn)) return + let messages = [...this.#state.messages] + const upsertTool = ( + callId: string, + name: string, + patch: Partial & { content?: string; success?: boolean } + ) => { + const { content, success, ...toolPatch } = patch + // Only this turn's tool messages are pending; a provider may reuse call ids across turns. + const i = messages.findIndex( + (m) => m.pending && m.role === 'tool' && m.tool?.callId === callId + ) + if (i >= 0) { + const existing = messages[i] + messages[i] = { + ...existing, + content: content ?? existing.content, + success: success ?? existing.success, + tool: { ...existing.tool!, ...toolPatch } + } + } else { + messages.push({ + id: `pending-${randomId()}`, + role: 'tool', + content: content ?? '', + success: success ?? true, + createdAt: now(), + pending: true, + tool: { callId, name, status: 'running', ...toolPatch } + }) + } + } + const appendAssistant = (text: string, reasoning: string) => { + const i = turn.assistantId + ? messages.findIndex((m) => m.id === turn.assistantId) + : -1 + if (i >= 0) { + const m = messages[i] + messages[i] = { + ...m, + content: m.content + text, + reasoning: reasoning ? (m.reasoning ?? '') + reasoning : m.reasoning + } + } else { + turn.assistantId = `pending-${randomId()}` + messages.push({ + id: turn.assistantId, + role: 'assistant', + content: text, + reasoning: reasoning || undefined, + success: true, + createdAt: now(), + pending: true + }) + } + } + for (const event of events) { + switch (event.type) { + case 'token_delta': + turn.streamedText = true + appendAssistant(event.content, '') + break + case 'reasoning_token_delta': + appendAssistant('', event.content) + break + case 'tool_call': + // The round's text is complete; text after the tool result is a new message. + turn.assistantId = undefined + upsertTool(event.call_id, event.function_name, { status: 'running' }) + break + case 'tool_call_arguments': + turn.assistantId = undefined + upsertTool(event.call_id, event.function_name, { arguments: event.arguments }) + break + case 'tool_execution': + turn.assistantId = undefined + upsertTool(event.call_id, event.function_name, { status: 'running' }) + break + case 'tool_result': + upsertTool(event.call_id, event.function_name, { + status: event.success ? 'success' : 'error', + result: event.result, + success: event.success, + // The same text Windmill persists for the tool message. + content: event.success + ? `Used ${event.function_name} tool` + : `Error executing ${event.function_name}` + }) + break + } + } + this.#set({ messages, status: 'streaming' }) + } + + async #finishTurn(turn: Turn, result: unknown, isNew: boolean): Promise { + if (!this.#turnActive(turn)) return + if (this.#state.history === 'server') { + turn.jobIds = await this.#turnJobIds(turn) + if (!this.#turnActive(turn)) return + const reconciled = await this.#reconcileTurn(turn) + if (!this.#turnActive(turn)) return + if (reconciled) { + this.#set({ status: 'idle' }) + this.#config.onFinish?.({ conversationId: turn.conversationId, jobId: turn.jobId, messages: this.#state.messages }) + if (isNew) await this.loadConversations().catch(() => {}) + return + } + // Server history just proved unreadable: the turn completes as local history. + } + let messages = this.#state.messages + let failed = false + if (isErrorResult(result)) { + // The envelope is also a legitimate result shape; the job's own status decides. + failed = await this.#api + .getCompletedResult(turn.jobId!, turn.controller.signal) + .then((r) => r.success === false) + .catch(() => true) + if (!this.#turnActive(turn)) return + if (failed) { + messages = [...messages, assistantMessage(errorResultMessage(result), false, turn.jobId)] + } + } + if (!failed && !turn.streamedText) { + const answer = extractChatAnswer(result) + if (answer !== undefined) { + messages = [...messages, assistantMessage(answer, true, turn.jobId)] + } + } + this.#set({ messages: finalized(messages), status: 'idle' }) + this.#persistLocal() + this.#config.onFinish?.({ conversationId: turn.conversationId, jobId: turn.jobId, messages: this.#state.messages }) + } + + /** + * Folds what the server persisted for the turn into the message list. The rows + * are written by the worker in their own transactions, each of which can trail + * the flow's completion, so a streamed message whose row hasn't landed stays and + * the list is re-read a few times before the rest is kept as streamed. + * Returns false when the server holds no answer for the turn: history fell back to + * local, the read was refused or kept failing, or no assistant row has landed. The + * caller then finishes the turn from the flow result, so an answer is never lost + * to history. Whether a row counts is read from the message list, not from what + * this read returned: the turn's polling may have merged the answer already. + */ + async #reconcileTurn(turn: Turn): Promise { + for (let attempt = 1; attempt <= RECONCILE_ATTEMPTS; attempt++) { + let rows: FlowConversationMessage[] + try { + rows = await this.#api.listMessages(turn.conversationId, { + afterSeq: this.#lastSeq(), + perPage: 100, + signal: turn.controller.signal + }) + } catch (e) { + if (isAbortError(e)) throw e + if (this.#fallBackToLocal(e)) return false + const refused = e instanceof WindmillApiError && (e.status === 401 || e.status === 403) + if (refused || attempt === RECONCILE_ATTEMPTS) return this.#answered(turn) + await sleep(RECONCILE_DELAY_MS, turn.controller.signal) + continue + } + if (!this.#turnActive(turn)) return true + this.#mergeRows(rows) + if (this.#answered(turn) && !this.#state.messages.some((m) => m.pending && m.content)) break + if (attempt < RECONCILE_ATTEMPTS) await sleep(RECONCILE_DELAY_MS, turn.controller.signal) + } + if (!this.#turnActive(turn)) return true + this.#set({ messages: finalized(this.#state.messages) }) + return this.#answered(turn) + } + + /** + * A persisted assistant message written by one of the turn's jobs follows the + * turn's user message. Tool rows alone are not an answer, and neither is a row + * from an earlier turn whose job outlived `stop()` (a token without `jobs:write` + * cannot cancel it), which can land after this turn's user row. + */ + #answered(turn: Turn): boolean { + const messages = this.#state.messages + const from = messages.findIndex((m) => m.id === turn.userMessageId) + const ownJob = (m: ChatMessage) => + turn.jobIds === undefined || (m.jobId !== undefined && turn.jobIds.has(m.jobId)) + return messages.some((m, i) => i > from && m.role === 'assistant' && m.seq !== undefined && ownJob(m)) + } + + /** + * The flow job plus every step job it ran, the failure and preprocessor steps + * included (a failure handler's answer is persisted under its own job). Unknown + * when the read fails. + */ + async #turnJobIds(turn: Turn): Promise | undefined> { + try { + const job = await this.#api.getFlowJob(turn.jobId!, turn.controller.signal) + const ids = new Set([turn.jobId!]) + const status = job.flow_status + for (const m of [...(status?.modules ?? []), status?.failure_module, status?.preprocessor_module]) { + if (m?.job) ids.add(m.job) + for (const j of m?.flow_jobs ?? []) ids.add(j) + } + return ids + } catch (e) { + if (isAbortError(e)) throw e + return undefined + } + } + + #failTurn(turn: Turn, e: unknown): void { + if (!this.#turnActive(turn)) return + const error = toError(e) + this.#set({ + messages: [...finalized(this.#state.messages), assistantMessage(error.message, false, turn.jobId)], + status: 'error', + error + }) + this.#persistLocal() + this.#config.onError?.(error, { conversationId: turn.conversationId, jobId: turn.jobId }) + } + + /** Before the answer streams, earlier steps may already have persisted messages. */ + #startPolling(turn: Turn): () => void { + let stopped = false + const { signal } = turn.controller + const loop = async () => { + while (!stopped) { + try { + await sleep(POLL_INTERVAL_MS, signal) + } catch { + return + } + if (stopped) return + try { + const rows = await this.#api.listMessages(turn.conversationId, { + afterSeq: this.#lastSeq(), + perPage: 100, + signal + }) + if (!stopped && this.#turnActive(turn)) this.#mergeRows(rows) + } catch { + // transient; the completion reconciliation catches up + } + } + } + void loop() + return () => { + stopped = true + } + } + + async #syncFromServer(conversationId: string): Promise { + const rows = await this.#api.listMessages(conversationId, { + afterSeq: this.#lastSeq(), + perPage: 100 + }) + if (this.#turn || this.#state.conversationId !== conversationId) return + this.#mergeRows(rows) + this.#set({ messages: finalized(this.#state.messages) }) + } + + /** + * Folds persisted rows into the message list. A row standing for a message the + * client already shows (same role and text; for a tool, the same tool name, since + * the server words a failure differently) takes its place under the client's id + * and keeps what only the stream knew: reasoning, call id, arguments, result. + * Other rows append in server order. Nothing is dropped: a streamed message + * outlives a row that never lands. + */ + #mergeRows(rows: FlowConversationMessage[]): void { + if (rows.length === 0) return + const messages = [...this.#state.messages] + const known = new Set(messages.map((m) => m.serverId ?? m.id)) + for (const row of rows.map(fromRow)) { + if (known.has(row.id)) continue + known.add(row.id) + const i = messages.findIndex( + (m) => + m.seq === undefined && + m.role === row.role && + (m.content === row.content || (row.tool !== undefined && m.tool?.name === row.tool.name)) + ) + if (i >= 0) { + const m = messages[i] + messages[i] = { + ...row, + id: m.id, + reasoning: m.reasoning ?? row.reasoning, + tool: m.tool ? { ...m.tool, status: row.tool?.status ?? m.tool.status } : row.tool + } + } else { + messages.push(row) + } + } + this.#set({ messages }) + } + + #lastSeq(): number | undefined { + let last: number | undefined + for (const m of this.#state.messages) { + if (m.seq !== undefined && (last === undefined || m.seq > last)) last = m.seq + } + return last + } + + /** Writes the current messages to local history; the conversation entry itself is `#rememberConversation`'s. */ + #persistLocal(): void { + clearTimeout(this.#persistTimer) + this.#persistTimer = undefined + const id = this.#state.conversationId + if (this.#state.history !== 'local' || !id) return + this.#local.saveMessages(id, this.#state.messages) + } + + /** + * Puts the current conversation at the head of local history. Only a turn moves + * a conversation there: merely viewing one must not reorder the list. + */ + #rememberConversation(): void { + const id = this.#state.conversationId + if (this.#state.history !== 'local' || !id) return + const conversation = this.#state.conversations.find((c) => c.id === id) + if (conversation) this.#local.upsertConversation(conversation) + } + + /** Whether an unreadable server history should silently become local history. */ + #fallBackToLocal(e: unknown): boolean { + if (this.#state.history !== 'server' || this.#config.historyExplicit) return false + if (e instanceof WindmillApiError && (e.status === 401 || e.status === 403)) { + this.#set({ history: 'local' }) + // The conversation now lives in the browser; list it there like one started local. + this.#rememberConversation() + return true + } + return false + } + + #turnActive(turn: Turn): boolean { + return this.#turn === turn && this.#state.conversationId === turn.conversationId + } + + /** Stops following the current answer; the flow itself keeps running. */ + #detachTurn(): void { + const turn = this.#turn + if (!turn) return + this.#turn = undefined + turn.controller.abort() + } + + /** + * Leaves the current conversation (for another one, or because the page goes + * away). A turn still in flight is detached and what it showed so far is kept, + * written out now rather than on the debounce that may never fire. + */ + #leaveConversation(): void { + if (this.#turn) { + this.#detachTurn() + this.#set({ messages: finalized(this.#state.messages), status: 'idle' }) + } + if (this.#persistTimer) this.#persistLocal() + } + + #set(patch: Partial): void { + this.#state = { ...this.#state, ...patch } + for (const listener of this.#listeners) listener(this.#state) + if (this.#state.history === 'local' && this.#state.conversationId) { + clearTimeout(this.#persistTimer) + this.#persistTimer = setTimeout(() => this.#persistLocal(), PERSIST_DEBOUNCE_MS) + } + } +} + +function fromRow(row: FlowConversationMessage): ChatMessage { + const toolName = + row.message_type === 'tool' + ? /^Used (.+) tool$/.exec(row.content)?.[1] ?? /^Error executing (.+)$/.exec(row.content)?.[1] + : undefined + const success = row.success ?? true + return { + id: row.id, + serverId: row.id, + role: row.message_type, + content: row.content, + success, + createdAt: row.created_at, + jobId: row.job_id ?? undefined, + stepName: row.step_name ?? undefined, + pending: false, + seq: row.created_seq, + tool: toolName ? { name: toolName, status: success ? 'success' : 'error' } : undefined + } +} + +function fromConversation(row: FlowConversation): Conversation { + return { + id: row.id, + title: row.title ?? undefined, + createdAt: row.created_at, + updatedAt: row.updated_at + } +} + +function assistantMessage(content: string, success: boolean, jobId: string | undefined): ChatMessage { + return { + id: `local-${randomId()}`, + role: 'assistant', + content, + success, + createdAt: now(), + jobId, + pending: false + } +} + +function finalized(messages: ChatMessage[]): ChatMessage[] { + return messages.some((m) => m.pending) + ? messages.map((m) => (m.pending ? { ...m, pending: false } : m)) + : messages +} + +function toError(e: unknown): Error { + return e instanceof Error ? e : new Error(String(e)) +} diff --git a/chat-sdk/src/config.ts b/chat-sdk/src/config.ts new file mode 100644 index 0000000000..88dc06781a --- /dev/null +++ b/chat-sdk/src/config.ts @@ -0,0 +1,81 @@ +import type { ChatOptions, FetchLike, HistoryMode, StorageLike, TokenSource } from './types' + +export interface ResolvedConfig { + flowPath: string + baseUrl: string + workspace: string + token: TokenSource | undefined + history: HistoryMode + /** The caller chose `history`; a failing server history is then an error, not a fallback. */ + historyExplicit: boolean + inputs: Record + fetch: FetchLike | undefined + storage: StorageLike | undefined + storageKey: string | undefined + pageSize: number + onFinish: ChatOptions['onFinish'] + onError: ChatOptions['onError'] +} + +export interface RawAppContext { + baseUrl: string + workspace: string + /** The viewer's SDK token in a sandboxed raw app; the session cookie otherwise. */ + token?: string +} + +/** + * The Windmill a raw app bundle runs in. A sandboxed app gets `window.process.env` + * from its wrapper once the viewer consented to the declared SDK scopes; an + * unsandboxed one runs on the Windmill origin with the viewer's session and only + * has `window.ctx`. + */ +export function detectRawApp(): RawAppContext | undefined { + const g = globalThis as { + process?: { env?: Record } + ctx?: { workspace?: unknown } + location?: { origin?: string } + } + const env = g.process?.env + if (env?.WM_RAW_APP === 'true' && env.WM_TOKEN && env.BASE_URL && env.WM_WORKSPACE) { + return { baseUrl: env.BASE_URL, workspace: env.WM_WORKSPACE, token: env.WM_TOKEN } + } + const workspace = g.ctx?.workspace + const origin = g.location?.origin + if (typeof workspace === 'string' && workspace && origin && origin !== 'null') { + return { baseUrl: origin, workspace } + } + return undefined +} + +export function resolveConfig(options: ChatOptions): ResolvedConfig { + if (!options.flowPath) throw new Error('windmill-chat: flowPath is required') + const explicitToken = options.token !== undefined + const detected = + options.baseUrl && options.workspace && explicitToken ? undefined : detectRawApp() + const baseUrl = options.baseUrl ?? detected?.baseUrl + const workspace = options.workspace ?? detected?.workspace + if (!baseUrl || !workspace) { + throw new Error( + 'windmill-chat: pass baseUrl and workspace. They are only detected inside a raw app: an unsandboxed one on the Windmill origin, or a sandboxed one whose policy declares frontend SDK scopes.' + ) + } + // The raw app's token belongs to its own Windmill; it never travels to another origin. + const token = + options.token ?? (options.baseUrl === undefined || options.baseUrl === detected?.baseUrl ? detected?.token : undefined) + return { + flowPath: options.flowPath, + baseUrl, + workspace, + token, + history: options.history ?? (explicitToken ? 'local' : 'server'), + historyExplicit: options.history !== undefined, + inputs: options.inputs ?? {}, + fetch: options.fetch, + storage: options.storage, + storageKey: options.storageKey, + pageSize: options.pageSize ?? 50, + onFinish: options.onFinish, + onError: options.onError + } +} diff --git a/chat-sdk/src/follow.ts b/chat-sdk/src/follow.ts new file mode 100644 index 0000000000..90631bbc8f --- /dev/null +++ b/chat-sdk/src/follow.ts @@ -0,0 +1,54 @@ +import type { WindmillChatApi } from './api' +import { createStreamEventParser, type AgentStreamEvent } from './stream' +import { abortError, sleep } from './utils' + +const RECONNECT_DELAY_MS = 300 + +export type FollowEvent = + /** Agent events decoded from the job's result stream; empty when a chunk ended mid-line. */ + | { type: 'stream'; events: AgentStreamEvent[] } + | { type: 'completed'; result: unknown } + +/** + * Follows a job to completion across the server's stream timeouts: every + * connection resumes from the last `stream_offset`, so no delta is repeated and + * the flow is never re-run. `onOffset` reports each offset so a caller can + * resume later from another connection (see the AI SDK transport). + */ +export async function* followJob( + api: WindmillChatApi, + jobId: string, + options: { signal?: AbortSignal; streamOffset?: number; onOffset?: (offset: number) => void } = {} +): AsyncGenerator { + const parser = createStreamEventParser() + let offset = options.streamOffset + while (true) { + let timedOut = false + for await (const update of api.streamJob(jobId, { streamOffset: offset, signal: options.signal })) { + if (update.type === 'ping') continue + if (update.type === 'timeout') { + timedOut = true + break + } + if (update.type === 'error') throw new Error(update.error) + if (update.type === 'notfound') throw new Error(`Job ${jobId} not found`) + if (update.stream_offset !== undefined) { + offset = update.stream_offset + options.onOffset?.(offset) + } + if (update.new_result_stream) { + yield { type: 'stream', events: parser.push(update.new_result_stream) } + } + if (update.completed) { + const rest = parser.flush() + if (rest.length > 0) yield { type: 'stream', events: rest } + yield { type: 'completed', result: update.only_result } + return + } + } + if (options.signal?.aborted) throw abortError() + // The server closes the connection after its timeout; a dropped connection looks + // the same minus the event. Either way the offset lets the next one resume. + if (!timedOut) await sleep(RECONNECT_DELAY_MS, options.signal) + } +} diff --git a/chat-sdk/src/history.ts b/chat-sdk/src/history.ts new file mode 100644 index 0000000000..485dd056cf --- /dev/null +++ b/chat-sdk/src/history.ts @@ -0,0 +1,90 @@ +import type { ChatMessage, Conversation, StorageLike } from './types' + +export interface LocalHistory { + listConversations(): Conversation[] + getMessages(conversationId: string): ChatMessage[] + upsertConversation(conversation: Conversation): void + saveMessages(conversationId: string, messages: ChatMessage[]): void + deleteConversation(conversationId: string): void +} + +interface Snapshot { + v: 1 + conversations: Conversation[] + messages: Record +} + +const MAX_CONVERSATIONS = 100 + +/** + * Conversation history in the browser, for a credential shared by every visitor + * (server history would show them each other's chats). One key per flow, and per + * `storageKey` when the caller sets one; every operation re-reads the store so + * several tabs stay consistent. + */ +export function createLocalHistory(storage: StorageLike | undefined, key: string): LocalHistory { + const store = storage ?? defaultStorage() + const read = (): Snapshot => { + try { + const raw = store.getItem(key) + if (raw) { + const parsed = JSON.parse(raw) as Snapshot + if (parsed && parsed.v === 1) return parsed + } + } catch { + // unreadable: start over + } + return { v: 1, conversations: [], messages: {} } + } + const write = (snapshot: Snapshot) => { + try { + store.setItem(key, JSON.stringify(snapshot)) + } catch { + // quota or private mode: the page keeps working from memory + } + } + return { + listConversations: () => read().conversations, + getMessages: (id) => read().messages[id] ?? [], + upsertConversation(conversation) { + const s = read() + const rest = s.conversations.filter((c) => c.id !== conversation.id) + s.conversations = [conversation, ...rest] + for (const dropped of s.conversations.splice(MAX_CONVERSATIONS)) { + delete s.messages[dropped.id] + } + write(s) + }, + saveMessages(id, messages) { + const s = read() + s.messages[id] = messages.map((m) => ({ ...m, pending: false })) + write(s) + }, + deleteConversation(id) { + const s = read() + s.conversations = s.conversations.filter((c) => c.id !== id) + delete s.messages[id] + write(s) + } + } +} + +function defaultStorage(): StorageLike { + try { + const ls = globalThis.localStorage + if (ls) { + const probe = '__windmill_chat_probe__' + ls.setItem(probe, '1') + ls.removeItem(probe) + return ls + } + } catch { + // no localStorage (SSR, blocked storage): fall through + } + const memory = new Map() + return { + getItem: (k) => memory.get(k) ?? null, + setItem: (k, v) => void memory.set(k, v), + removeItem: (k) => void memory.delete(k) + } +} diff --git a/chat-sdk/src/index.ts b/chat-sdk/src/index.ts new file mode 100644 index 0000000000..353c814154 --- /dev/null +++ b/chat-sdk/src/index.ts @@ -0,0 +1,29 @@ +export { createChat } from './chat' +export { detectRawApp, type RawAppContext } from './config' +export { + WindmillChatApi, + WindmillApiError, + readServerSentEvents, + type WindmillChatApiOptions, + type FlowConversation, + type FlowConversationMessage, + type JobUpdateEvent, + type CompletedJobResult +} from './api' +export { parseStreamEvents, createStreamEventParser, type AgentStreamEvent } from './stream' +export { followJob, type FollowEvent } from './follow' +export { extractChatAnswer, conversationIdFor } from './utils' +export type { + Chat, + ChatMessage, + ChatOptions, + ChatRole, + ChatState, + ChatStatus, + Conversation, + FetchLike, + HistoryMode, + StorageLike, + TokenSource, + ToolInvocation +} from './types' diff --git a/chat-sdk/src/react.ts b/chat-sdk/src/react.ts new file mode 100644 index 0000000000..48d3b1037c --- /dev/null +++ b/chat-sdk/src/react.ts @@ -0,0 +1,67 @@ +import { useEffect, useMemo, useRef, useSyncExternalStore } from 'react' +import { createChat } from './chat' +import type { Chat, ChatOptions, ChatState } from './types' + +export type UseWindmillChat = ChatState & + Pick< + Chat, + | 'sendMessage' + | 'stop' + | 'newConversation' + | 'selectConversation' + | 'loadConversations' + | 'deleteConversation' + | 'loadOlderMessages' + > & { chat: Chat } + +/** + * A chat on a chat-mode flow. The chat is created once per `flowPath`, `baseUrl`, + * `workspace`, `history`, `storageKey` and credential, and destroyed on unmount. A + * credential change is a new user, whose chat must not carry the previous one's + * state: a different token string, or a switch between no token, a token string + * and a token function, all recreate it. A token function is read through a ref + * on every call, so a new closure per render changes what the next call runs and + * nothing else; pass a `storageKey` per user when local history must not be shared. + * The callbacks and `inputs` are read the same way: the latest render's values go + * with the next message. + */ +export function useWindmillChat(options: ChatOptions): UseWindmillChat { + const latest = useRef(options) + latest.current = options + const credential = + typeof options.token === 'function' ? 'fn' : typeof options.token === 'string' ? `str:${options.token}` : 'none' + const chat = useMemo( + () => + createChat({ + ...options, + token: + typeof options.token === 'function' + ? () => { + const token = latest.current.token + return typeof token === 'function' ? token() : (token ?? '') + } + : options.token, + onFinish: (turn) => latest.current.onFinish?.(turn), + onError: (error, turn) => latest.current.onError?.(error, turn) + }), + // eslint-disable-next-line react-hooks/exhaustive-deps + [options.flowPath, options.baseUrl, options.workspace, options.history, options.storageKey, credential] + ) + useEffect(() => () => chat.destroy(), [chat]) + const state = useSyncExternalStore(chat.subscribe, chat.getState, chat.getState) + return useMemo( + () => ({ + ...state, + chat, + sendMessage: (text, options) => + chat.sendMessage(text, { ...options, inputs: { ...latest.current.inputs, ...options?.inputs } }), + stop: chat.stop, + newConversation: chat.newConversation, + selectConversation: chat.selectConversation, + loadConversations: chat.loadConversations, + deleteConversation: chat.deleteConversation, + loadOlderMessages: chat.loadOlderMessages + }), + [state, chat] + ) +} diff --git a/chat-sdk/src/stream.ts b/chat-sdk/src/stream.ts new file mode 100644 index 0000000000..c6969bb7b4 --- /dev/null +++ b/chat-sdk/src/stream.ts @@ -0,0 +1,65 @@ +/** The events an AI agent step streams, one JSON object per line of the job's result stream. */ +export type AgentStreamEvent = + | { type: 'token_delta'; content: string } + | { type: 'reasoning_token_delta'; content: string } + | { type: 'tool_call'; call_id: string; function_name: string } + | { type: 'tool_call_arguments'; call_id: string; function_name: string; arguments: string } + | { type: 'tool_execution'; call_id: string; function_name: string } + | { + type: 'tool_result' + call_id: string + function_name: string + result: string + success: boolean + } + +const KNOWN_TYPES = new Set([ + 'token_delta', + 'reasoning_token_delta', + 'tool_call', + 'tool_call_arguments', + 'tool_execution', + 'tool_result' +]) + +/** + * Incremental parser for the `new_result_stream` chunks of a job update. A chunk is + * not guaranteed to end on a line boundary, so an incomplete last line waits for the + * next `push` (or `flush` once the job completes). + */ +export function createStreamEventParser() { + let pending = '' + return { + push(chunk: string): AgentStreamEvent[] { + pending += chunk + const lastNewline = pending.lastIndexOf('\n') + if (lastNewline === -1) return [] + const complete = pending.slice(0, lastNewline) + pending = pending.slice(lastNewline + 1) + return parseStreamEvents(complete) + }, + flush(): AgentStreamEvent[] { + const rest = pending + pending = '' + return parseStreamEvents(rest) + } + } +} + +/** Parses complete NDJSON lines; lines that aren't agent events are skipped. */ +export function parseStreamEvents(ndjson: string): AgentStreamEvent[] { + const events: AgentStreamEvent[] = [] + for (const line of ndjson.split('\n')) { + const trimmed = line.trim() + if (!trimmed) continue + try { + const parsed = JSON.parse(trimmed) + if (parsed && typeof parsed === 'object' && KNOWN_TYPES.has(parsed.type)) { + events.push(parsed as AgentStreamEvent) + } + } catch { + // not an agent event + } + } + return events +} diff --git a/chat-sdk/src/types.ts b/chat-sdk/src/types.ts new file mode 100644 index 0000000000..31b44cdd8f --- /dev/null +++ b/chat-sdk/src/types.ts @@ -0,0 +1,123 @@ +export type ChatRole = 'user' | 'assistant' | 'tool' | 'system' + +/** + * - `idle`: ready for a message + * - `submitted`: the message was sent, no answer has started streaming yet + * - `streaming`: the answer is arriving + * - `error`: the last turn failed; `error` holds why. Sending again is allowed. + */ +export type ChatStatus = 'idle' | 'submitted' | 'streaming' | 'error' + +/** + * Where conversation history lives. + * - `server`: Windmill's conversation store. Each Windmill user only sees their own + * conversations, so use it with the viewer's own session or a per-user token. + * - `local`: the browser's storage. Right for a token shared by every visitor. + * - `none`: nothing is kept beyond the current page. + */ +export type HistoryMode = 'server' | 'local' | 'none' + +export interface ToolInvocation { + callId?: string + name: string + /** The arguments the model passed, as a JSON string. */ + arguments?: string + result?: string + status: 'running' | 'success' | 'error' +} + +export interface ChatMessage { + id: string + role: ChatRole + content: string + /** The model's reasoning summary, when the provider streams one. */ + reasoning?: string + /** Set on `tool` messages that came from the live stream. */ + tool?: ToolInvocation + success: boolean + createdAt: string + jobId?: string + /** The flow step that produced the message. */ + stepName?: string + /** True while the message is optimistic or still streaming. */ + pending: boolean + /** Id of the persisted row once the server has it; `id` itself never changes, so list keys stay stable. */ + serverId?: string + /** The server's cursor for a persisted message; unset for one created on the client. */ + seq?: number +} + +export interface Conversation { + id: string + title: string | undefined + createdAt: string + updatedAt: string +} + +export interface ChatState { + conversationId: string | undefined + messages: ChatMessage[] + status: ChatStatus + error: Error | undefined + conversations: Conversation[] + /** Where history is read from. Starts as configured; drops from `server` to `local` when the credential cannot read conversations. */ + history: HistoryMode + loadingMessages: boolean + hasMoreMessages: boolean +} + +export type TokenSource = string | (() => string | Promise) + +export type StorageLike = Pick + +export type FetchLike = (input: string | URL | Request, init?: RequestInit) => Promise + +export interface ChatOptions { + /** Path of a deployed flow with chat mode enabled, e.g. `f/support/assistant`. */ + flowPath: string + /** Windmill origin, e.g. `https://app.windmill.dev`. Detected inside a raw app. */ + baseUrl?: string + /** Detected inside a raw app. */ + workspace?: string + /** + * A Windmill token, or a function returning one (called before every request, so it + * can fetch a short-lived token from your backend). Omit it to use the viewer's + * session: the cookie on the Windmill origin, or a sandboxed raw app's SDK token. + */ + token?: TokenSource + /** Defaults to `server` with the viewer's session and `local` with an explicit token. */ + history?: HistoryMode + /** Extra flow inputs sent with every message, next to `user_message`. */ + inputs?: Record + fetch?: FetchLike + /** Backing store for `local` history. Defaults to `localStorage`. */ + storage?: StorageLike + /** + * Namespace for `local` history, e.g. the signed-in user's id. Local history is + * per browser, per flow; without this, users sharing a browser share it. + */ + storageKey?: string + /** Messages fetched per page of server history. */ + pageSize?: number + /** Called once a turn has its answer (a failed flow included: its error is the answer). */ + onFinish?: (turn: { conversationId: string; jobId?: string; messages: ChatMessage[] }) => void + /** Called when a turn could not run or be followed; `state.error` holds the same error. */ + onError?: (error: Error, turn: { conversationId: string; jobId?: string }) => void +} + +export interface Chat { + getState(): ChatState + /** Calls `listener` now and on every change; returns the unsubscribe function (Svelte store contract). */ + subscribe(listener: (state: ChatState) => void): () => void + /** Sends a message in the current conversation, starting one when there is none. Resolves when the answer is complete. */ + sendMessage(text: string, options?: { inputs?: Record }): Promise + /** Stops following the answer and asks Windmill to cancel the run. */ + stop(): Promise + newConversation(): void + selectConversation(conversationId: string): Promise + loadConversations(options?: { page?: number; perPage?: number }): Promise + deleteConversation(conversationId: string): Promise + loadOlderMessages(): Promise + /** Stops background work (stream, polling) and writes local history out. The chat stays usable. */ + destroy(): void +} diff --git a/chat-sdk/src/utils.ts b/chat-sdk/src/utils.ts new file mode 100644 index 0000000000..fe7e5d8960 --- /dev/null +++ b/chat-sdk/src/utils.ts @@ -0,0 +1,134 @@ +export function randomId(): string { + const c = globalThis.crypto + if (c?.randomUUID) return c.randomUUID() + // `randomUUID` needs a secure context; a plain http dev origin has `getRandomValues` only. + const bytes = new Uint8Array(16) + c.getRandomValues(bytes) + return formatUuid(bytes, 4) +} + +const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i + +export function isUuid(value: string): boolean { + return UUID_RE.test(value) +} + +/** + * The Windmill conversation id for an arbitrary chat id. A UUID is used as is; + * anything else (an AI SDK chat id, for instance) maps to the same UUID every time, + * so a page can reopen its conversation without storing a second id. Hashed in plain + * JS: `crypto.subtle` only exists in secure contexts, and the mapping must not depend + * on the origin's scheme. + */ +export function conversationIdFor(chatId: string): string { + if (isUuid(chatId)) return chatId.toLowerCase() + const bytes = new TextEncoder().encode(`windmill-chat:${chatId}`) + const out = new Uint8Array(16) + for (const [i, seed] of [0xcbf29ce484222325n, 0x84222325cbf29ce4n].entries()) { + let h = fnv1a64(bytes, seed) + for (let b = 7; b >= 0; b--) { + out[i * 8 + b] = Number(h & 0xffn) + h >>= 8n + } + } + return formatUuid(out, 5) +} + +function fnv1a64(bytes: Uint8Array, seed: bigint): bigint { + let h = seed + for (const byte of bytes) { + h ^= BigInt(byte) + h = (h * 0x100000001b3n) & 0xffffffffffffffffn + } + return h +} + +function formatUuid(bytes: Uint8Array, version: 4 | 5): string { + bytes[6] = (bytes[6] & 0x0f) | (version << 4) + bytes[8] = (bytes[8] & 0x3f) | 0x80 + const hex = Array.from(bytes, (b) => b.toString(16).padStart(2, '0')).join('') + return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}` +} + +export function parseJsonOr(text: string | undefined): unknown { + if (text === undefined) return undefined + try { + return JSON.parse(text) + } catch { + return text + } +} + +export function now(): string { + return new Date().toISOString() +} + +/** Same rule as the server: the first message, cut to 25 characters. */ +export function conversationTitle(firstMessage: string): string { + const chars = Array.from(firstMessage) + return chars.length > 25 ? `${chars.slice(0, 25).join('')}...` : firstMessage +} + +export function sleep(ms: number, signal?: AbortSignal): Promise { + return new Promise((resolve, reject) => { + if (signal?.aborted) return reject(abortError()) + const onAbort = () => { + clearTimeout(timer) + reject(abortError()) + } + const timer = setTimeout(() => { + signal?.removeEventListener('abort', onAbort) + resolve() + }, ms) + signal?.addEventListener('abort', onAbort, { once: true }) + }) +} + +export function abortError(): Error { + return new DOMException('The operation was aborted', 'AbortError') +} + +export function isAbortError(e: unknown): boolean { + return e instanceof Error && e.name === 'AbortError' +} + +/** + * The text a chat shows for a flow result, following what Windmill persists as the + * assistant message when the last step is not an AI agent: `windmill_chat_answer` + * when the result carries one (null means no message), an agent result's `output`, + * a string as is, anything else as JSON. + */ +export function extractChatAnswer(result: unknown): string | undefined { + if (result === null || result === undefined) return undefined + if (typeof result === 'string') return result + if (typeof result === 'object' && !Array.isArray(result)) { + const obj = result as Record + if ('windmill_chat_answer' in obj) return formatAnswer(obj.windmill_chat_answer) + if ('output' in obj && Array.isArray(obj.messages)) return formatAnswer(obj.output) + } + return JSON.stringify(result, null, 2) +} + +function formatAnswer(value: unknown): string | undefined { + if (value === null || value === undefined) return undefined + return typeof value === 'string' ? value : JSON.stringify(value, null, 2) +} + +/** A completed job whose result is Windmill's error envelope. */ +export function isErrorResult(result: unknown): result is { error: Record } { + return ( + typeof result === 'object' && + result !== null && + 'error' in result && + typeof (result as { error: unknown }).error === 'object' && + (result as { error: unknown }).error !== null + ) +} + +export function errorResultMessage(result: { error: Record }): string { + const { message, name } = result.error + if (typeof message === 'string' && message) { + return typeof name === 'string' && name && name !== 'Error' ? `${name}: ${message}` : message + } + return JSON.stringify(result.error, null, 2) +} diff --git a/chat-sdk/test/ai-sdk-chat.test.ts b/chat-sdk/test/ai-sdk-chat.test.ts new file mode 100644 index 0000000000..a0bbb7b8a1 --- /dev/null +++ b/chat-sdk/test/ai-sdk-chat.test.ts @@ -0,0 +1,62 @@ +import { describe, expect, test } from 'bun:test' +import { Chat } from '@ai-sdk/react' +import { createWindmillChatTransport } from '../src/ai-sdk' +import { fetchMock, json, ndjson, sse, text } from './support' + +const FLOW = 'f/chat/agent' + +/** The AI SDK's own chunk processor consuming the transport, as `useChat` would. */ +describe('AI SDK Chat over the Windmill transport', () => { + test('builds the assistant message parts and settles to ready', async () => { + let jobs = 0 + const { fetch, calls } = fetchMock( + (c) => + c.method === 'POST' && c.url.pathname === `/api/w/ws/jobs/run/f/${FLOW}` ? text(`job-${++jobs}`) : undefined, + (c) => + c.url.pathname.endsWith('/getupdate_sse/job-1') + ? sse([ + { + type: 'update', + new_result_stream: ndjson( + { type: 'tool_call', call_id: 'c1', function_name: 'lookup' }, + { type: 'tool_call_arguments', call_id: 'c1', function_name: 'lookup', arguments: '{"q":1}' }, + { type: 'tool_result', call_id: 'c1', function_name: 'lookup', result: '42', success: true }, + { type: 'reasoning_token_delta', content: 'so ' }, + { type: 'token_delta', content: 'The answer ' }, + { type: 'token_delta', content: 'is 42' } + ), + stream_offset: 6, + completed: true, + only_result: { output: 'The answer is 42', messages: [] } + } + ]) + : undefined, + (c) => + c.url.pathname.endsWith('/getupdate_sse/job-2') + ? sse([{ type: 'update', completed: true, only_result: { error: { message: 'boom' } } }]) + : undefined, + (c) => + c.url.pathname.endsWith('/completed/get_result_maybe/job-2') ? json({ completed: true, success: false }) : undefined + ) + const transport = createWindmillChatTransport({ baseUrl: 'http://wm.test', workspace: 'ws', flowPath: FLOW, token: 'tok', fetch }) + const chat = new Chat({ id: 'e2e-chat', transport }) + + await chat.sendMessage({ text: 'what is it?' }) + + expect(chat.status).toBe('ready') + expect(chat.messages.map((m) => m.role)).toEqual(['user', 'assistant']) + const parts = chat.messages[1].parts + expect(parts.map((p) => p.type)).toEqual(['dynamic-tool', 'reasoning', 'text']) + expect(parts[0]).toMatchObject({ toolName: 'lookup', toolCallId: 'c1', state: 'output-available', input: { q: 1 }, output: 42 }) + expect(parts[1]).toMatchObject({ type: 'reasoning', text: 'so ', state: 'done' }) + expect(parts[2]).toMatchObject({ type: 'text', text: 'The answer is 42', state: 'done' }) + // Both turns of the chat ran in the same Windmill conversation. + const memoryIds = calls.filter((c) => c.method === 'POST').map((c) => c.url.searchParams.get('memory_id')) + expect(memoryIds[0]).toBe(transport.conversationId('e2e-chat')) + + await chat.sendMessage({ text: 'and now fail' }) + expect(chat.status).toBe('error') + expect(chat.error?.message).toBe('boom') + expect(memoryIds.length === 1 || calls.filter((c) => c.method === 'POST')[1].url.searchParams.get('memory_id') === memoryIds[0]).toBe(true) + }) +}) diff --git a/chat-sdk/test/ai-sdk.test.ts b/chat-sdk/test/ai-sdk.test.ts new file mode 100644 index 0000000000..7f30f26edc --- /dev/null +++ b/chat-sdk/test/ai-sdk.test.ts @@ -0,0 +1,178 @@ +import { describe, expect, test } from 'bun:test' +import type { UIMessage, UIMessageChunk } from 'ai' +import { createWindmillChatTransport, toUIMessages } from '../src/ai-sdk' +import type { ChatMessage } from '../src/types' +import { fetchMock, json, ndjson, sse, text, type Route } from './support' + +const FLOW = 'f/chat/agent' +const run: Route = (c) => + c.method === 'POST' && c.url.pathname === `/api/w/ws/jobs/run/f/${FLOW}` ? text('job-1') : undefined +const streamPath = '/api/w/ws/jobs_u/getupdate_sse/job-1' + +const userMessage = (text: string): UIMessage => ({ id: 'u1', role: 'user', parts: [{ type: 'text', text }] }) + +async function collect(stream: ReadableStream): Promise { + const chunks: UIMessageChunk[] = [] + const reader = stream.getReader() + while (true) { + const { value, done } = await reader.read() + if (done) return chunks + chunks.push(value) + } +} + +describe('createWindmillChatTransport', () => { + test('maps an agent turn to AI SDK chunks and derives the conversation from the chat id', async () => { + const { fetch, calls } = fetchMock(run, (c) => + c.url.pathname === streamPath + ? sse([ + { + type: 'update', + new_result_stream: ndjson( + { type: 'reasoning_token_delta', content: 'think' }, + { type: 'token_delta', content: 'Let me ' }, + { type: 'tool_call', call_id: 'c1', function_name: 'lookup' }, + { type: 'tool_call_arguments', call_id: 'c1', function_name: 'lookup', arguments: '{"q":1}' }, + { type: 'tool_execution', call_id: 'c1', function_name: 'lookup' } + ), + stream_offset: 5 + }, + { + type: 'update', + new_result_stream: ndjson( + { type: 'tool_result', call_id: 'c1', function_name: 'lookup', result: '{"answer":42}', success: true }, + { type: 'token_delta', content: '42' } + ), + stream_offset: 7, + completed: true, + only_result: { output: '42', messages: [] } + } + ]) + : undefined + ) + const transport = createWindmillChatTransport({ + baseUrl: 'http://wm.test', + workspace: 'ws', + flowPath: FLOW, + token: 'tok', + inputs: { tone: 'kind' }, + fetch + }) + const chunks = await collect( + await transport.sendMessages({ + trigger: 'submit-message', + chatId: 'chat-abc', + messageId: undefined, + messages: [userMessage('what is it?')], + abortSignal: undefined, + body: { locale: 'fr' } + }) + ) + + const runCall = calls.find((c) => c.method === 'POST')! + expect(runCall.body).toEqual({ tone: 'kind', locale: 'fr', user_message: 'what is it?' }) + expect(runCall.url.searchParams.get('memory_id')).toBe(transport.conversationId('chat-abc')) + expect(transport.conversationId('chat-abc')).toMatch(/^[0-9a-f]{8}-[0-9a-f]{4}-5[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/) + expect(transport.conversationId('chat-abc')).toBe(transport.conversationId('chat-abc')) + expect(transport.conversationId('chat-abd')).not.toBe(transport.conversationId('chat-abc')) + expect(transport.conversationId('4D6E5C8C-2C3B-4E1A-9F31-0B2E6F1C9D10')).toBe( + '4d6e5c8c-2c3b-4e1a-9f31-0b2e6f1c9d10' + ) + + const shape = chunks.map((c) => ('delta' in c ? `${c.type}:${c.delta}` : c.type)) + expect(shape).toEqual([ + 'start', + 'reasoning-start', + 'reasoning-delta:think', + 'text-start', + 'text-delta:Let me ', + 'reasoning-end', + 'text-end', + 'tool-input-start', + 'tool-input-available', + 'tool-output-available', + 'text-start', + 'text-delta:42', + 'text-end', + 'finish' + ]) + expect(chunks.find((c) => c.type === 'tool-input-available')).toMatchObject({ + toolCallId: 'c1', + toolName: 'lookup', + input: { q: 1 }, + dynamic: true + }) + expect(chunks.find((c) => c.type === 'tool-output-available')).toMatchObject({ output: { answer: 42 } }) + // The job finished, so there is nothing to reconnect to. + expect(await transport.reconnectToStream({ chatId: 'chat-abc' })).toBeNull() + }) + + test('answers from the flow result when nothing streamed, and reports a failed flow as an error', async () => { + let turn = 0 + const { fetch } = fetchMock( + run, + (c) => { + if (c.url.pathname !== streamPath) return undefined + turn++ + return sse([ + turn === 1 + ? { type: 'update', completed: true, only_result: { windmill_chat_answer: 'From a script' } } + : { type: 'update', completed: true, only_result: { error: { name: 'ExecutionErr', message: 'boom' } } } + ]) + }, + (c) => + c.url.pathname === '/api/w/ws/jobs_u/completed/get_result_maybe/job-1' + ? json({ completed: true, success: false }) + : undefined + ) + const transport = createWindmillChatTransport({ baseUrl: 'http://wm.test', workspace: 'ws', flowPath: FLOW, fetch }) + const send = () => + transport.sendMessages({ + trigger: 'submit-message', + chatId: 'c', + messageId: undefined, + messages: [userMessage('hi')], + abortSignal: undefined + }) + const first = await collect(await send()) + expect(first.map((c) => ('delta' in c ? c.delta : c.type))).toEqual(['start', 'text-start', 'From a script', 'text-end', 'finish']) + const second = await collect(await send()) + expect(second.map((c) => c.type)).toEqual(['start', 'error']) + expect(second[1]).toMatchObject({ errorText: 'ExecutionErr: boom' }) + }) + + test('refuses attachments with a clear error', async () => { + const transport = createWindmillChatTransport({ baseUrl: 'http://wm.test', workspace: 'ws', flowPath: FLOW, fetch: fetchMock().fetch }) + await expect( + transport.sendMessages({ + trigger: 'submit-message', + chatId: 'c', + messageId: undefined, + messages: [{ id: 'u', role: 'user', parts: [{ type: 'file', mediaType: 'image/png', url: 'data:...' }] }], + abortSignal: undefined + }) + ).rejects.toThrow('attachments are not supported') + }) +}) + +describe('toUIMessages', () => { + test('folds a turn into one assistant message with reasoning, tool and text parts', () => { + const base = { success: true, createdAt: '2026-01-01T00:00:00Z', pending: false } + const messages: ChatMessage[] = [ + { ...base, id: 'u1', role: 'user', content: 'hi' }, + { ...base, id: 't1', role: 'tool', content: 'Used lookup tool', tool: { callId: 'c1', name: 'lookup', status: 'success', arguments: '{"q":1}', result: '42' } }, + { ...base, id: 'a1', role: 'assistant', content: 'The answer is 42', reasoning: 'hmm' }, + { ...base, id: 'u2', role: 'user', content: 'thanks' }, + { ...base, id: 't2', role: 'tool', content: 'Error executing lookup', success: false, tool: { name: 'lookup', status: 'error' } } + ] + const ui = toUIMessages(messages) + expect(ui.map((m) => [m.id, m.role, m.parts.map((p) => p.type)])).toEqual([ + ['u1', 'user', ['text']], + ['t1', 'assistant', ['dynamic-tool', 'reasoning', 'text']], + ['u2', 'user', ['text']], + ['t2', 'assistant', ['dynamic-tool']] + ]) + expect(ui[1].parts[0]).toMatchObject({ toolCallId: 'c1', toolName: 'lookup', state: 'output-available', input: { q: 1 }, output: 42 }) + expect(ui[3].parts[0]).toMatchObject({ state: 'output-error', errorText: 'Error executing lookup' }) + }) +}) diff --git a/chat-sdk/test/assistant-ui.test.ts b/chat-sdk/test/assistant-ui.test.ts new file mode 100644 index 0000000000..5641dc2907 --- /dev/null +++ b/chat-sdk/test/assistant-ui.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, test } from 'bun:test' +import { groupTurns, toThreadMessage } from '../src/assistant-ui' +import type { ChatMessage } from '../src/types' + +const base = { success: true, createdAt: '2026-01-01T00:00:00Z', pending: false } + +describe('assistant-ui conversion', () => { + test('groups rows into turns and renders tool calls as content parts', () => { + const messages: ChatMessage[] = [ + { ...base, id: 'u1', role: 'user', content: 'hi' }, + { ...base, id: 't1', role: 'tool', content: 'Used lookup tool', tool: { callId: 'c1', name: 'lookup', status: 'success', arguments: '{"q":1}', result: '42' } }, + { ...base, id: 'a1', role: 'assistant', content: 'The answer is 42', reasoning: 'hmm' }, + { ...base, id: 'u2', role: 'user', content: 'again' }, + { ...base, id: 'a2', role: 'assistant', content: 'partial', pending: true } + ] + const turns = groupTurns(messages) + expect(turns.map((t) => [t.id, t.role, t.messages.length])).toEqual([ + ['u1', 'user', 1], + ['t1', 'assistant', 2], + ['u2', 'user', 1], + ['a2', 'assistant', 1] + ]) + const answer = toThreadMessage(turns[1]) + expect(answer).toMatchObject({ id: 't1', role: 'assistant', status: { type: 'complete', reason: 'stop' } }) + expect(answer.content).toEqual([ + { type: 'tool-call', toolCallId: 'c1', toolName: 'lookup', args: { q: 1 }, argsText: '{"q":1}', result: 42, isError: false }, + { type: 'reasoning', text: 'hmm' }, + { type: 'text', text: 'The answer is 42' } + ]) + expect(toThreadMessage(turns[3]).status).toEqual({ type: 'running' }) + expect(toThreadMessage(turns[0])).toMatchObject({ role: 'user', content: [{ type: 'text', text: 'hi' }] }) + }) + + test('marks a failed answer as incomplete', () => { + const [turn] = groupTurns([{ ...base, id: 'a', role: 'assistant', content: 'boom', success: false }]) + expect(toThreadMessage(turn).status).toEqual({ type: 'incomplete', reason: 'error', error: 'boom' }) + }) +}) diff --git a/chat-sdk/test/chat.test.ts b/chat-sdk/test/chat.test.ts new file mode 100644 index 0000000000..93f8752dcc --- /dev/null +++ b/chat-sdk/test/chat.test.ts @@ -0,0 +1,671 @@ +import { describe, expect, test } from 'bun:test' +import { createChat } from '../src/chat' +import type { ChatOptions } from '../src/types' +import { fetchMock, json, memoryStorage, messageRow, ndjson, sse, sseTimed, text, type Route } from './support' + +const BASE = 'http://wm.test' +const FLOW = 'f/chat/agent' + +const run: Route = (c) => + c.method === 'POST' && c.url.pathname === `/api/w/ws/jobs/run/f/${FLOW}` ? text('job-1') : undefined + +const streamPath = '/api/w/ws/jobs_u/getupdate_sse/job-1' + +function options(extra: Partial, fetch: ChatOptions['fetch']): ChatOptions { + return { flowPath: FLOW, baseUrl: BASE, workspace: 'ws', fetch, storage: memoryStorage(), ...extra } +} + +describe('createChat with local history', () => { + test('streams text and tool calls, then finalizes and persists the turn', async () => { + const storage = memoryStorage() + const { fetch, calls } = fetchMock(run, (c) => + c.url.pathname === streamPath + ? sse([ + { type: 'ping' }, + { + type: 'update', + new_result_stream: ndjson( + { type: 'token_delta', content: 'Let me ' }, + { type: 'tool_call', call_id: 'c1', function_name: 'lookup' }, + { type: 'tool_call_arguments', call_id: 'c1', function_name: 'lookup', arguments: '{"q":1}' } + ), + stream_offset: 3, + flow_stream_job_id: 'agent-job' + }, + { + type: 'update', + new_result_stream: ndjson( + { type: 'tool_result', call_id: 'c1', function_name: 'lookup', result: '42', success: true }, + { type: 'token_delta', content: 'The answer is 42' } + ), + stream_offset: 5, + completed: true, + only_result: { output: 'The answer is 42', messages: [] } + } + ]) + : undefined + ) + const chat = createChat(options({ token: 'tok', storage }, fetch)) + const statuses: string[] = [] + chat.subscribe((s) => statuses.push(s.status)) + + await chat.sendMessage('what is the answer?', { inputs: { locale: 'fr' } }) + + const runCall = calls.find((c) => c.method === 'POST')! + expect(runCall.url.searchParams.get('memory_id')).toBe(chat.getState().conversationId!) + expect(runCall.body).toEqual({ locale: 'fr', user_message: 'what is the answer?' }) + expect(runCall.headers.authorization).toBe('Bearer tok') + + const state = chat.getState() + expect(state.status).toBe('idle') + expect(statuses).toContain('submitted') + expect(statuses).toContain('streaming') + expect(state.messages.map((m) => [m.role, m.content, m.pending])).toEqual([ + ['user', 'what is the answer?', false], + ['assistant', 'Let me ', false], + ['tool', 'Used lookup tool', false], + ['assistant', 'The answer is 42', false] + ]) + expect(state.messages[2].tool).toEqual({ + callId: 'c1', + name: 'lookup', + status: 'success', + arguments: '{"q":1}', + result: '42' + }) + expect(state.conversations).toHaveLength(1) + expect(state.conversations[0].title).toBe('what is the answer?') + + const reloaded = createChat(options({ token: 'tok', storage }, fetch)) + await reloaded.loadConversations() + expect(reloaded.getState().conversations.map((c) => c.id)).toEqual([state.conversationId!]) + await reloaded.selectConversation(state.conversationId!) + expect(reloaded.getState().messages.map((m) => m.content)).toEqual( + state.messages.map((m) => m.content) + ) + }) + + test('a tool call id reused by a later turn gets its own message', async () => { + const toolTurn = () => + sse([ + { + type: 'update', + new_result_stream: ndjson( + { type: 'tool_call', call_id: 'same-id', function_name: 'lookup' }, + { type: 'tool_result', call_id: 'same-id', function_name: 'lookup', result: '1', success: true }, + { type: 'token_delta', content: 'done' } + ), + stream_offset: 3, + completed: true, + only_result: { output: 'done', messages: [] } + } + ]) + const { fetch } = fetchMock(run, (c) => (c.url.pathname === streamPath ? toolTurn() : undefined)) + const chat = createChat(options({ token: 'tok' }, fetch)) + await chat.sendMessage('one') + await chat.sendMessage('two') + expect(chat.getState().messages.map((m) => m.role)).toEqual([ + 'user', + 'tool', + 'assistant', + 'user', + 'tool', + 'assistant' + ]) + }) + + test('resumes after a stream timeout from the last offset without re-running the flow', async () => { + let streamCalls = 0 + const { fetch, calls } = fetchMock(run, (c) => { + if (c.url.pathname !== streamPath) return undefined + streamCalls++ + if (streamCalls === 1) { + return sse([ + { type: 'update', new_result_stream: ndjson({ type: 'token_delta', content: 'Hel' }), stream_offset: 1 }, + { type: 'timeout' } + ]) + } + return sse([ + { + type: 'update', + new_result_stream: ndjson({ type: 'token_delta', content: 'lo' }), + stream_offset: 2, + completed: true, + only_result: 'Hello' + } + ]) + }) + const chat = createChat(options({ token: 'tok' }, fetch)) + await chat.sendMessage('hi') + + expect(calls.filter((c) => c.method === 'POST')).toHaveLength(1) + const streams = calls.filter((c) => c.url.pathname === streamPath) + expect(streams).toHaveLength(2) + expect(streams[0].url.searchParams.get('stream_offset')).toBeNull() + expect(streams[1].url.searchParams.get('stream_offset')).toBe('1') + expect(chat.getState().messages.map((m) => m.content)).toEqual(['hi', 'Hello']) + }) + + test('derives the answer from the flow result when nothing streamed', async () => { + const { fetch } = fetchMock(run, (c) => + c.url.pathname === streamPath + ? sse([{ type: 'update', completed: true, only_result: { windmill_chat_answer: 'From a script' } }]) + : undefined + ) + const chat = createChat(options({ token: 'tok' }, fetch)) + await chat.sendMessage('hi') + const [, answer] = chat.getState().messages + expect(answer.role).toBe('assistant') + expect(answer.content).toBe('From a script') + expect(answer.jobId).toBe('job-1') + }) + + test('renders a successful result that merely looks like an error envelope', async () => { + const result = { error: { message: 'domain data' } } + const { fetch } = fetchMock( + run, + (c) => (c.url.pathname === streamPath ? sse([{ type: 'update', completed: true, only_result: result }]) : undefined), + (c) => + c.url.pathname === '/api/w/ws/jobs_u/completed/get_result_maybe/job-1' + ? json({ completed: true, success: true, result }) + : undefined + ) + const chat = createChat(options({ token: 'tok' }, fetch)) + await chat.sendMessage('hi') + expect(chat.getState().messages[1]).toMatchObject({ + role: 'assistant', + success: true, + content: JSON.stringify(result, null, 2) + }) + }) + + test('reports a failed flow as an unsuccessful assistant message', async () => { + const { fetch } = fetchMock( + run, + (c) => + c.url.pathname === streamPath + ? sse([ + { + type: 'update', + completed: true, + only_result: { error: { name: 'ExecutionErr', message: 'boom' } } + } + ]) + : undefined, + (c) => + c.url.pathname === '/api/w/ws/jobs_u/completed/get_result_maybe/job-1' + ? json({ completed: true, success: false, result: { error: { message: 'boom' } } }) + : undefined + ) + const chat = createChat(options({ token: 'tok' }, fetch)) + await chat.sendMessage('hi') + const state = chat.getState() + expect(state.status).toBe('idle') + expect(state.messages[1]).toMatchObject({ role: 'assistant', success: false, content: 'ExecutionErr: boom' }) + }) +}) + +describe('createChat with server history', () => { + test('replaces the optimistic turn with the persisted rows', async () => { + const { fetch, calls } = fetchMock( + run, + (c) => + c.url.pathname === streamPath + ? sse([ + { + type: 'update', + new_result_stream: ndjson( + { type: 'reasoning_token_delta', content: 'hmm' }, + { type: 'token_delta', content: 'Hello' } + ), + stream_offset: 2, + completed: true, + only_result: { output: 'Hello', messages: [] } + } + ]) + : undefined, + (c) => + c.method === 'GET' && c.url.pathname.endsWith('/messages') + ? json([ + messageRow(11, 'user', 'hi'), + messageRow(12, 'assistant', 'Hello', { step_name: 'AI Agent', job_id: 'agent-job' }) + ]) + : undefined, + (c) => + c.url.pathname === '/api/w/ws/flow_conversations/list' + ? json([ + { + id: chatId, + workspace_id: 'ws', + flow_path: FLOW, + title: 'hi', + created_at: '2026-01-01T00:00:00Z', + updated_at: '2026-01-01T00:00:01Z', + created_by: 'admin' + } + ]) + : undefined + ) + const chat = createChat(options({}, fetch)) + let chatId = '' + const unsubscribe = chat.subscribe((s) => { + chatId = s.conversationId ?? chatId + }) + await chat.sendMessage('hi') + unsubscribe() + + const state = chat.getState() + expect(state.history).toBe('server') + expect(state.messages.map((m) => [m.serverId, m.role, m.content, m.pending])).toEqual([ + ['row-11', 'user', 'hi', false], + ['row-12', 'assistant', 'Hello', false] + ]) + // Ids stay the client's, so list keys never remount; the server id rides alongside. + expect(state.messages.map((m) => m.id.startsWith('pending-'))).toEqual([true, true]) + expect(state.messages[1]).toMatchObject({ reasoning: 'hmm', stepName: 'AI Agent', jobId: 'agent-job', seq: 12 }) + expect(state.conversations.map((c) => c.id)).toEqual([chatId]) + + const messagesCall = calls.find((c) => c.url.pathname.endsWith('/messages'))! + expect(messagesCall.url.pathname).toBe(`/api/w/ws/flow_conversations/${chatId}/messages`) + expect(messagesCall.headers.authorization).toBeUndefined() + }) + + test('keeps the streamed answer until its row lands, even when a tool row lands first', async () => { + let messageFetches = 0 + const { fetch } = fetchMock( + run, + (c) => + c.url.pathname === streamPath + ? sse([ + { + type: 'update', + new_result_stream: ndjson( + { type: 'tool_call', call_id: 'c1', function_name: 'lookup' }, + { type: 'tool_result', call_id: 'c1', function_name: 'lookup', result: '1', success: true }, + { type: 'token_delta', content: 'Final answer' } + ), + stream_offset: 3, + completed: true, + only_result: { output: 'Final answer', messages: [] } + } + ]) + : undefined, + (c) => { + if (!c.url.pathname.endsWith('/messages')) return undefined + messageFetches++ + // The assistant row is written by a task that trails the tool's. + return json( + messageFetches === 1 + ? [messageRow(21, 'user', 'hi'), messageRow(22, 'tool', 'Used lookup tool')] + : [messageRow(23, 'assistant', 'Final answer')] + ) + }, + (c) => (c.url.pathname === '/api/w/ws/flow_conversations/list' ? json([]) : undefined) + ) + const chat = createChat(options({}, fetch)) + await chat.sendMessage('hi') + expect(messageFetches).toBe(2) + expect(chat.getState().messages.map((m) => [m.serverId, m.role, m.content, m.pending])).toEqual([ + ['row-21', 'user', 'hi', false], + ['row-22', 'tool', 'Used lookup tool', false], + ['row-23', 'assistant', 'Final answer', false] + ]) + expect(chat.getState().messages[1].tool).toMatchObject({ callId: 'c1', result: '1', status: 'success' }) + }) + + test('finishes the turn from the flow result when history falls back mid-turn', async () => { + const storage = memoryStorage() + const { fetch } = fetchMock( + run, + (c) => + c.url.pathname === streamPath + ? sse([{ type: 'update', completed: true, only_result: { windmill_chat_answer: 'From a script' } }]) + : undefined, + (c) => (c.url.pathname.includes('/flow_conversations/') ? text('forbidden', 403) : undefined) + ) + const chat = createChat(options({ storage }, fetch)) + await chat.sendMessage('hi') + const state = chat.getState() + expect(state.history).toBe('local') + expect(state.status).toBe('idle') + expect(state.messages.map((m) => [m.role, m.content])).toEqual([ + ['user', 'hi'], + ['assistant', 'From a script'] + ]) + const stored = JSON.parse([...storage.data.values()][0]) + expect(stored.messages[state.conversationId!]).toHaveLength(2) + }) + + test('appends later pages of conversations', async () => { + const row = (id: string) => ({ + id, + workspace_id: 'ws', + flow_path: FLOW, + title: id, + created_at: '2026-01-01T00:00:00Z', + updated_at: '2026-01-01T00:00:00Z', + created_by: 'admin' + }) + const { fetch } = fetchMock((c) => + c.url.pathname === '/api/w/ws/flow_conversations/list' + ? json(c.url.searchParams.get('page') === '2' ? [row('c2')] : [row('c1')]) + : undefined + ) + const chat = createChat(options({}, fetch)) + await chat.loadConversations() + await chat.loadConversations({ page: 2 }) + expect(chat.getState().conversations.map((c) => c.id)).toEqual(['c1', 'c2']) + }) + + test('a turn started right after stop() is not touched by the stop sync', async () => { + let jobs = 0 + const { fetch } = fetchMock( + (c) => (c.method === 'POST' && c.url.pathname.includes('/jobs/run/f/') ? text(`job-${++jobs}`) : undefined), + (c) => + c.url.pathname.endsWith('/getupdate_sse/job-1') + ? sse([{ type: 'update', new_result_stream: ndjson({ type: 'token_delta', content: 'slow...' }), stream_offset: 1 }]) + : undefined, + (c) => + c.url.pathname.endsWith('/getupdate_sse/job-2') + ? sse([ + { + type: 'update', + new_result_stream: ndjson( + { type: 'tool_call', call_id: 'c2', function_name: 'lookup' }, + { type: 'tool_result', call_id: 'c2', function_name: 'lookup', result: '1', success: true }, + { type: 'token_delta', content: 'second' } + ), + stream_offset: 3, + completed: true, + only_result: { output: 'second', messages: [] } + } + ]) + : undefined, + (c) => (c.url.pathname.includes('/queue/cancel/') ? text('ok') : undefined), + (c) => + c.url.pathname.endsWith('/messages') + ? json([messageRow(31, 'user', 'first'), messageRow(32, 'user', 'second question'), messageRow(33, 'tool', 'Used lookup tool'), messageRow(34, 'assistant', 'second')]) + : undefined, + (c) => (c.url.pathname === '/api/w/ws/flow_conversations/list' ? json([]) : undefined) + ) + const chat = createChat(options({}, fetch)) + const first = chat.sendMessage('first') + // The stream of job-1 never completes: the connection just ends, so the turn keeps waiting. + await new Promise((r) => setTimeout(r, 50)) + const stopped = chat.stop() + await first + const second = chat.sendMessage('second question') + await stopped + await second + const roles = chat.getState().messages.map((m) => `${m.role}${m.pending ? '*' : ''}`) + expect(roles).toEqual(['user', 'assistant', 'user', 'tool', 'assistant']) + expect(chat.getState().messages.filter((m) => m.role === 'tool')).toHaveLength(1) + }) + + test('answers from the flow result when only the user row has been persisted', async () => { + let reads = 0 + const { fetch } = fetchMock( + run, + (c) => + c.url.pathname === streamPath + ? sse([{ type: 'update', completed: true, only_result: { windmill_chat_answer: 'From a script' } }]) + : undefined, + (c) => (c.url.pathname.endsWith('/messages') ? (reads++, json([messageRow(41, 'user', 'hi')])) : undefined), + (c) => (c.url.pathname === '/api/w/ws/flow_conversations/list' ? json([]) : undefined) + ) + const chat = createChat(options({}, fetch)) + await chat.sendMessage('hi') + expect(reads).toBeGreaterThan(1) + expect(chat.getState().messages.map((m) => [m.role, m.content, m.serverId])).toEqual([ + ['user', 'hi', 'row-41'], + ['assistant', 'From a script', undefined] + ]) + }) + + test('an answer the poller merged before completion is not appended again', async () => { + const { fetch } = fetchMock( + run, + (c) => + c.url.pathname === streamPath + ? sseTimed([{ type: 'update' }, 1400, { type: 'update', completed: true, only_result: { windmill_chat_answer: 'From a script' } }]) + : undefined, + (c) => + c.url.pathname.endsWith('/messages') + ? json([messageRow(51, 'user', 'hi'), messageRow(52, 'assistant', 'From a script')]) + : undefined, + (c) => (c.url.pathname === '/api/w/ws/flow_conversations/list' ? json([]) : undefined) + ) + const chat = createChat(options({}, fetch)) + await chat.sendMessage('hi') + expect(chat.getState().messages.map((m) => [m.role, m.content, m.serverId])).toEqual([ + ['user', 'hi', 'row-51'], + ['assistant', 'From a script', 'row-52'] + ]) + }) + + test('a tool row alone is not the answer of a turn that streamed no text', async () => { + let reads = 0 + const { fetch } = fetchMock( + run, + (c) => + c.url.pathname === streamPath + ? sse([{ type: 'update', completed: true, only_result: { output: 'Answer', messages: [] } }]) + : undefined, + (c) => + c.url.pathname.endsWith('/messages') + ? json(++reads === 1 ? [messageRow(61, 'user', 'hi'), messageRow(62, 'tool', 'Used lookup tool')] : [messageRow(63, 'assistant', 'Answer')]) + : undefined, + (c) => (c.url.pathname === '/api/w/ws/flow_conversations/list' ? json([]) : undefined) + ) + const chat = createChat(options({}, fetch)) + await chat.sendMessage('hi') + expect(reads).toBe(2) + expect(chat.getState().messages.map((m) => [m.role, m.content, m.serverId])).toEqual([ + ['user', 'hi', 'row-61'], + ['tool', 'Used lookup tool', 'row-62'], + ['assistant', 'Answer', 'row-63'] + ]) + }) + + test('a late answer from a stopped job is not taken as the next turn answer', async () => { + let jobs = 0 + let reads = 0 + const { fetch } = fetchMock( + (c) => (c.method === 'POST' && c.url.pathname.includes('/jobs/run/f/') ? text(`job-${++jobs}`) : undefined), + // job-1 never completes: the connection just ends, so the turn keeps waiting. + (c) => (c.url.pathname.endsWith('/getupdate_sse/job-1') ? sse([{ type: 'update' }]) : undefined), + (c) => + c.url.pathname.endsWith('/getupdate_sse/job-2') + ? sse([{ type: 'update', completed: true, only_result: { windmill_chat_answer: 'second answer' } }]) + : undefined, + // The run-only token cannot cancel: job-1 keeps running after stop(). + (c) => (c.url.pathname.includes('/queue/cancel/') ? text('forbidden', 400) : undefined), + (c) => + c.url.pathname.endsWith('/jobs_u/get/job-2') + ? json({ flow_status: { modules: [{ job: 'step-2' }] } }) + : undefined, + // Read 1 is stop()'s sync; the stopped job's answer lands after the second user row. + (c) => + c.url.pathname.endsWith('/messages') + ? json( + ++reads === 1 + ? [messageRow(71, 'user', 'first')] + : reads === 2 + ? [messageRow(72, 'user', 'second'), messageRow(73, 'assistant', 'first answer, late', { job_id: 'step-1' })] + : [messageRow(74, 'assistant', 'second answer', { job_id: 'step-2' })] + ) + : undefined, + (c) => (c.url.pathname === '/api/w/ws/flow_conversations/list' ? json([]) : undefined) + ) + const chat = createChat(options({}, fetch)) + const first = chat.sendMessage('first') + await new Promise((r) => setTimeout(r, 50)) + await chat.stop() + await first + await chat.sendMessage('second') + expect(chat.getState().messages.map((m) => [m.role, m.content, m.serverId])).toEqual([ + ['user', 'first', 'row-71'], + ['user', 'second', 'row-72'], + ['assistant', 'first answer, late', 'row-73'], + ['assistant', 'second answer', 'row-74'] + ]) + }) + + test('a failure handler answer is attributed to the turn', async () => { + let reads = 0 + const { fetch } = fetchMock( + run, + (c) => + c.url.pathname === streamPath + ? sse([{ type: 'update', completed: true, only_result: { error: { name: 'ExecutionErr', message: 'boom' } } }]) + : undefined, + (c) => + c.url.pathname.endsWith('/jobs_u/get/job-1') + ? json({ flow_status: { modules: [{ job: 'step-1' }], failure_module: { job: 'handler-1' } } }) + : undefined, + (c) => + c.url.pathname.endsWith('/messages') + ? (reads++, json([messageRow(81, 'user', 'hi'), messageRow(82, 'assistant', 'Sorry: boom', { job_id: 'handler-1', success: false })])) + : undefined, + (c) => (c.url.pathname === '/api/w/ws/flow_conversations/list' ? json([]) : undefined) + ) + const chat = createChat(options({}, fetch)) + await chat.sendMessage('hi') + expect(reads).toBe(1) + expect(chat.getState().messages.map((m) => [m.role, m.content, m.success, m.serverId])).toEqual([ + ['user', 'hi', true, 'row-81'], + ['assistant', 'Sorry: boom', false, 'row-82'] + ]) + }) + + test('deleting the current local conversation mid-turn leaves nothing behind', async () => { + const storage = memoryStorage() + const { fetch } = fetchMock(run, (c) => + c.url.pathname === streamPath + ? sse([{ type: 'update', new_result_stream: ndjson({ type: 'token_delta', content: 'partial' }), stream_offset: 1 }]) + : undefined + ) + const chat = createChat(options({ token: 'tok', storage }, fetch)) + const turn = chat.sendMessage('hello') + await new Promise((r) => setTimeout(r, 300)) + const id = chat.getState().conversationId! + await chat.deleteConversation(id) + await turn + await new Promise((r) => setTimeout(r, 400)) + expect(chat.getState().conversations).toEqual([]) + await chat.selectConversation(id) + expect(chat.getState().messages).toEqual([]) + expect([...storage.data.values()].join('')).not.toContain('hello') + }) + + test('viewing an older local conversation does not reorder history', async () => { + const storage = memoryStorage() + const { fetch } = fetchMock(run, (c) => + c.url.pathname === streamPath ? sse([{ type: 'update', completed: true, only_result: 'ok' }]) : undefined + ) + const chat = createChat(options({ token: 'tok', storage }, fetch)) + await chat.sendMessage('older') + const older = chat.getState().conversationId! + chat.newConversation() + await chat.sendMessage('newer') + const newer = chat.getState().conversationId! + await chat.selectConversation(older) + await new Promise((r) => setTimeout(r, 400)) + const again = createChat(options({ token: 'tok', storage }, fetch)) + expect((await again.loadConversations()).map((c) => c.id)).toEqual([newer, older]) + }) + + test('destroying the chat mid-turn leaves it idle', async () => { + const { fetch } = fetchMock(run, (c) => + c.url.pathname === streamPath + ? sse([{ type: 'update', new_result_stream: ndjson({ type: 'token_delta', content: 'partial' }), stream_offset: 1 }]) + : undefined + ) + const chat = createChat(options({ token: 'tok' }, fetch)) + const turn = chat.sendMessage('hello') + await new Promise((r) => setTimeout(r, 50)) + expect(chat.getState().status).toBe('streaming') + chat.destroy() + await turn + expect(chat.getState().status).toBe('idle') + expect(chat.getState().messages.every((m) => !m.pending)).toBe(true) + }) + + test('destroying the chat during a local turn keeps what it showed', async () => { + const storage = memoryStorage() + const { fetch } = fetchMock(run, (c) => + c.url.pathname === streamPath + ? sse([{ type: 'update', new_result_stream: ndjson({ type: 'token_delta', content: 'partial' }), stream_offset: 1 }]) + : undefined + ) + const chat = createChat(options({ token: 'tok', storage }, fetch)) + const turn = chat.sendMessage('hello') + await new Promise((r) => setTimeout(r, 50)) + const id = chat.getState().conversationId! + chat.destroy() + await turn + const again = createChat(options({ token: 'tok', storage }, fetch)) + await again.loadConversations() + expect(again.getState().conversations.map((c) => c.id)).toEqual([id]) + await again.selectConversation(id) + expect(again.getState().messages.map((m) => [m.role, m.content, m.pending])).toEqual([ + ['user', 'hello', false], + ['assistant', 'partial', false] + ]) + }) + + test('switching conversations keeps what a local turn showed so far', async () => { + const storage = memoryStorage() + const { fetch } = fetchMock(run, (c) => + c.url.pathname === streamPath + ? sse([{ type: 'update', new_result_stream: ndjson({ type: 'token_delta', content: 'partial' }), stream_offset: 1 }]) + : undefined + ) + const chat = createChat(options({ token: 'tok', storage }, fetch)) + const turn = chat.sendMessage('hello') + await new Promise((r) => setTimeout(r, 50)) + const id = chat.getState().conversationId! + chat.newConversation() + await turn + expect(chat.getState().messages).toEqual([]) + await chat.selectConversation(id) + expect(chat.getState().messages.map((m) => [m.role, m.content, m.pending])).toEqual([ + ['user', 'hello', false], + ['assistant', 'partial', false] + ]) + }) + + test('answers from the flow result when server history keeps failing', async () => { + const { fetch } = fetchMock( + run, + (c) => + c.url.pathname === streamPath + ? sse([{ type: 'update', completed: true, only_result: { windmill_chat_answer: 'From a script' } }]) + : undefined, + (c) => (c.url.pathname.includes('/flow_conversations/') ? text('down', 503) : undefined) + ) + const chat = createChat(options({ history: 'server' }, fetch)) + await chat.sendMessage('hi') + const state = chat.getState() + expect(state.history).toBe('server') + expect(state.status).toBe('idle') + expect(state.messages.map((m) => [m.role, m.content])).toEqual([ + ['user', 'hi'], + ['assistant', 'From a script'] + ]) + }) + + test('falls back to local history when the credential cannot read conversations', async () => { + const { fetch } = fetchMock((c) => + c.url.pathname === '/api/w/ws/flow_conversations/list' ? text('forbidden', 403) : undefined + ) + const chat = createChat(options({}, fetch)) + expect(chat.getState().history).toBe('server') + await chat.loadConversations() + expect(chat.getState().history).toBe('local') + + const explicit = createChat(options({ history: 'server' }, fetch)) + await expect(explicit.loadConversations()).rejects.toThrow('403') + expect(explicit.getState().history).toBe('server') + }) +}) diff --git a/chat-sdk/test/config.test.ts b/chat-sdk/test/config.test.ts new file mode 100644 index 0000000000..2289f20c89 --- /dev/null +++ b/chat-sdk/test/config.test.ts @@ -0,0 +1,49 @@ +import { afterEach, describe, expect, test } from 'bun:test' +import { resolveConfig } from '../src/config' + +const g = globalThis as { process?: unknown; ctx?: unknown; location?: unknown } +const originalProcess = g.process + +afterEach(() => { + g.process = originalProcess + delete g.ctx + delete g.location +}) + +describe('resolveConfig', () => { + test('explicit token defaults history to local; a session defaults to server', () => { + const base = { flowPath: 'f/a/b', baseUrl: 'http://wm.test/', workspace: 'ws' } + expect(resolveConfig({ ...base, token: 't' }).history).toBe('local') + expect(resolveConfig(base).history).toBe('server') + expect(resolveConfig({ ...base, token: 't', history: 'server' }).historyExplicit).toBe(true) + }) + + test('reads the sandboxed raw app env the wrapper injects', () => { + g.process = { + env: { WM_RAW_APP: 'true', WM_TOKEN: 'sdk-token', BASE_URL: 'http://wm.test', WM_WORKSPACE: 'ws' } + } + const config = resolveConfig({ flowPath: 'f/a/b' }) + expect(config).toMatchObject({ baseUrl: 'http://wm.test', workspace: 'ws', token: 'sdk-token', history: 'server' }) + }) + + test('keeps the raw app token off another instance', () => { + g.process = { + env: { WM_RAW_APP: 'true', WM_TOKEN: 'sdk-token', BASE_URL: 'http://wm.test', WM_WORKSPACE: 'ws' } + } + expect(resolveConfig({ flowPath: 'f/a/b', baseUrl: 'http://other.test', workspace: 'ws' }).token).toBeUndefined() + expect(resolveConfig({ flowPath: 'f/a/b', baseUrl: 'http://wm.test' }).token).toBe('sdk-token') + }) + + test('reads the unsandboxed raw app context and uses the page origin', () => { + g.ctx = { ctx: { username: 'admin' }, workspace: 'ws' } + g.location = { origin: 'http://wm.test' } + const config = resolveConfig({ flowPath: 'f/a/b' }) + expect(config).toMatchObject({ baseUrl: 'http://wm.test', workspace: 'ws', token: undefined }) + }) + + test('refuses an opaque origin without an SDK token', () => { + g.ctx = { workspace: 'ws' } + g.location = { origin: 'null' } + expect(() => resolveConfig({ flowPath: 'f/a/b' })).toThrow('frontend SDK scopes') + }) +}) diff --git a/chat-sdk/test/react.test.tsx b/chat-sdk/test/react.test.tsx new file mode 100644 index 0000000000..542d274e9e --- /dev/null +++ b/chat-sdk/test/react.test.tsx @@ -0,0 +1,80 @@ +import { GlobalRegistrator } from '@happy-dom/global-registrator' +// Test files share one process: the DOM globals must not outlive this file. +GlobalRegistrator.register() + +import { afterAll, describe, expect, test } from 'bun:test' +import React, { act } from 'react' +import { createRoot, type Root } from 'react-dom/client' +import { useWindmillChat, type UseWindmillChat } from '../src/react' +import type { ChatOptions } from '../src/types' +import { fetchMock, memoryStorage } from './support' + +afterAll(() => GlobalRegistrator.unregister()) + +const base: ChatOptions = { + flowPath: 'f/chat/agent', + baseUrl: 'http://wm.test', + workspace: 'ws', + fetch: fetchMock().fetch, + storage: memoryStorage() +} + +/** Renders the hook and hands back what it returned, rerendering with new options on demand. */ +function mountHook() { + let latest: UseWindmillChat | undefined + const Probe = (props: ChatOptions) => { + latest = useWindmillChat(props) + return null + } + const root: Root = createRoot(document.createElement('div')) + const render = (props: ChatOptions) => { + act(() => root.render()) + return latest! + } + return { render, unmount: () => act(() => root.unmount()) } +} + +describe('useWindmillChat', () => { + test('a credential change is a new chat; a new closure for the same credential is not', () => { + const { render, unmount } = mountHook() + const a = render({ ...base, token: 'user-a' }).chat + expect(render({ ...base, token: 'user-a' }).chat).toBe(a) + const b = render({ ...base, token: 'user-b' }).chat + expect(b).not.toBe(a) + + const fn1 = render({ ...base, token: () => 'fn-1' }).chat + expect(fn1).not.toBe(b) + expect(render({ ...base, token: () => 'fn-2' }).chat).toBe(fn1) + + const session = render({ ...base }).chat + expect(session).not.toBe(fn1) + const fn3 = render({ ...base, token: () => 'fn-3' }).chat + expect(fn3).not.toBe(session) + expect(render({ ...base, token: () => 'fn-3', storageKey: 'someone-else' }).chat).not.toBe(fn3) + unmount() + }) + + test('the latest inputs go with the next message', async () => { + const { fetch, calls } = fetchMock((c) => (c.method === 'POST' ? new Response('job-1') : undefined)) + const { render, unmount } = mountHook() + render({ ...base, fetch, token: 'tok', inputs: { docId: 'first' } }) + const hook = render({ ...base, fetch, token: 'tok', inputs: { docId: 'second' } }) + // The run's stream never answers here; only the request matters. + void hook.sendMessage('hi', { inputs: { extra: true } }).catch(() => {}) + await new Promise((r) => setTimeout(r, 20)) + expect(calls.find((c) => c.method === 'POST')?.body).toEqual({ docId: 'second', extra: true, user_message: 'hi' }) + unmount() + }) + + test('a token function is read through a ref, so the latest closure serves the next request', async () => { + const { fetch, calls } = fetchMock((c) => (c.url.pathname.includes('/flow_conversations/list') ? new Response('[]') : undefined)) + const { render, unmount } = mountHook() + const first = render({ ...base, fetch, history: 'server', token: () => 'first' }) + await act(() => first.loadConversations()) + const second = render({ ...base, fetch, history: 'server', token: () => 'second' }) + expect(second.chat).toBe(first.chat) + await act(() => second.loadConversations()) + expect(calls.map((c) => c.headers.authorization)).toEqual(['Bearer first', 'Bearer second']) + unmount() + }) +}) diff --git a/chat-sdk/test/stream.test.ts b/chat-sdk/test/stream.test.ts new file mode 100644 index 0000000000..84f2502133 --- /dev/null +++ b/chat-sdk/test/stream.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, test } from 'bun:test' +import { readServerSentEvents } from '../src/api' +import { createStreamEventParser, parseStreamEvents } from '../src/stream' +import { ndjson } from './support' + +describe('parseStreamEvents', () => { + test('keeps agent events and skips other lines', () => { + const events = parseStreamEvents( + ndjson( + { type: 'token_delta', content: 'Hi' }, + { type: 'reasoning_token_delta', content: 'thinking' }, + { type: 'something_else', content: 'x' }, + { type: 'tool_result', call_id: 'c1', function_name: 'lookup', result: '42', success: true } + ) + 'not json\n' + ) + expect(events.map((e) => e.type)).toEqual(['token_delta', 'reasoning_token_delta', 'tool_result']) + }) +}) + +describe('createStreamEventParser', () => { + test('holds an incomplete line until the rest arrives', () => { + const parser = createStreamEventParser() + const line = JSON.stringify({ type: 'token_delta', content: 'Hello' }) + expect(parser.push(line.slice(0, 10))).toEqual([]) + expect(parser.push(line.slice(10) + '\n' + '{"type":"token_delta",')).toEqual([ + { type: 'token_delta', content: 'Hello' } + ]) + expect(parser.push('"content":"!"}')).toEqual([]) + expect(parser.flush()).toEqual([{ type: 'token_delta', content: '!' }]) + }) +}) + +describe('readServerSentEvents', () => { + test('splits frames that straddle chunks and normalizes CRLF', async () => { + const chunks = ['data: {"a":1}\r\n\r\ndata: {"b"', ':2}\n\ndata: first\ndata: second\n\n', 'data: {"c":3}'] + const encoder = new TextEncoder() + const body = new ReadableStream({ + start(controller) { + for (const c of chunks) controller.enqueue(encoder.encode(c)) + controller.close() + } + }) + const frames: string[] = [] + for await (const data of readServerSentEvents(body)) frames.push(data) + expect(frames).toEqual(['{"a":1}', '{"b":2}', 'first\nsecond', '{"c":3}']) + }) + + test('keeps a CRLF split across chunks from ending the event', async () => { + const chunks = ['data: first\r', '\ndata: second\r\n\r\ndata: last\r'] + const encoder = new TextEncoder() + const body = new ReadableStream({ + start(controller) { + for (const c of chunks) controller.enqueue(encoder.encode(c)) + controller.close() + } + }) + const frames: string[] = [] + for await (const data of readServerSentEvents(body)) frames.push(data) + expect(frames).toEqual(['first\nsecond', 'last']) + }) +}) diff --git a/chat-sdk/test/support.ts b/chat-sdk/test/support.ts new file mode 100644 index 0000000000..c794e50f3d --- /dev/null +++ b/chat-sdk/test/support.ts @@ -0,0 +1,101 @@ +import type { FetchLike, StorageLike } from '../src/types' + +export interface RecordedCall { + method: string + url: URL + headers: Record + body: unknown +} + +export type Route = (call: RecordedCall) => Response | Promise | undefined + +/** A fetch whose responses come from the first route that answers; every call is recorded. */ +export function fetchMock(...routes: Route[]): { fetch: FetchLike; calls: RecordedCall[] } { + const calls: RecordedCall[] = [] + const fetch: FetchLike = async (input, init) => { + const url = new URL(typeof input === 'string' ? input : input instanceof URL ? input.href : input.url) + const call: RecordedCall = { + method: init?.method ?? 'GET', + url, + headers: Object.fromEntries( + Object.entries((init?.headers as Record) ?? {}).map(([k, v]) => [k.toLowerCase(), v]) + ), + body: typeof init?.body === 'string' ? JSON.parse(init.body) : undefined + } + calls.push(call) + for (const route of routes) { + const res = await route(call) + if (res) return res + } + return new Response(`no route for ${call.method} ${url.pathname}`, { status: 404 }) + } + return { fetch, calls } +} + +export function json(value: unknown, status = 200): Response { + return new Response(JSON.stringify(value), { + status, + headers: { 'content-type': 'application/json' } + }) +} + +export function text(value: string, status = 200): Response { + return new Response(value, { status }) +} + +/** A `text/event-stream` body carrying one `data:` frame per event. */ +export function sse(events: object[]): Response { + return new Response(events.map((e) => `data: ${JSON.stringify(e)}\n\n`).join(''), { + status: 200, + headers: { 'content-type': 'text/event-stream' } + }) +} + +/** Like `sse`, but a number in the list pauses that many milliseconds before the next frame. */ +export function sseTimed(events: (object | number)[]): Response { + const encoder = new TextEncoder() + const body = new ReadableStream({ + async start(controller) { + for (const e of events) { + if (typeof e === 'number') await new Promise((r) => setTimeout(r, e)) + else controller.enqueue(encoder.encode(`data: ${JSON.stringify(e)}\n\n`)) + } + controller.close() + } + }) + return new Response(body, { status: 200, headers: { 'content-type': 'text/event-stream' } }) +} + +export function ndjson(...events: object[]): string { + return events.map((e) => JSON.stringify(e)).join('\n') + '\n' +} + +export function memoryStorage(): StorageLike & { data: Map } { + const data = new Map() + return { + data, + getItem: (k) => data.get(k) ?? null, + setItem: (k, v) => void data.set(k, v), + removeItem: (k) => void data.delete(k) + } +} + +export function messageRow( + seq: number, + type: 'user' | 'assistant' | 'tool', + content: string, + extra: Record = {} +) { + return { + id: `row-${seq}`, + conversation_id: 'conv', + message_type: type, + content, + job_id: null, + created_at: '2026-01-01T00:00:00Z', + created_seq: seq, + step_name: null, + success: true, + ...extra + } +} diff --git a/chat-sdk/tsconfig.build.json b/chat-sdk/tsconfig.build.json new file mode 100644 index 0000000000..b8fc4fa324 --- /dev/null +++ b/chat-sdk/tsconfig.build.json @@ -0,0 +1,12 @@ +{ + "extends": "./tsconfig.json", + "include": ["src/**/*"], + "compilerOptions": { + "types": [], + "noEmit": false, + "declaration": true, + "emitDeclarationOnly": true, + "outDir": "dist", + "rootDir": "src" + } +} diff --git a/chat-sdk/tsconfig.json b/chat-sdk/tsconfig.json new file mode 100644 index 0000000000..fbfee1f0d5 --- /dev/null +++ b/chat-sdk/tsconfig.json @@ -0,0 +1,15 @@ +{ + "include": ["src/**/*", "test/**/*"], + "compilerOptions": { + "target": "ES2020", + "lib": ["ES2020", "DOM", "DOM.Iterable"], + "module": "ESNext", + "moduleResolution": "bundler", + "jsx": "react-jsx", + "types": ["bun"], + "strict": true, + "noEmit": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true + } +} diff --git a/chat-sdk/tsdown.config.ts b/chat-sdk/tsdown.config.ts new file mode 100644 index 0000000000..98b23c536e --- /dev/null +++ b/chat-sdk/tsdown.config.ts @@ -0,0 +1,9 @@ +import { defineConfig } from 'tsdown' + +export default defineConfig({ + entry: ['src/index.ts', 'src/react.ts', 'src/ai-sdk.ts', 'src/assistant-ui.ts'], + format: ['esm', 'cjs'], + dts: false, + external: ['react', 'ai', '@assistant-ui/react'], + target: 'es2020', +}) diff --git a/cli/src/guidance/skills.gen.ts b/cli/src/guidance/skills.gen.ts index 3f63574b83..f646b08f8c 100644 --- a/cli/src/guidance/skills.gen.ts +++ b/cli/src/guidance/skills.gen.ts @@ -5907,6 +5907,19 @@ An app can be demoed by recording a session: every interaction becomes a step ca Apply it to customer data, internal notes and anything else a viewer of the demo should not see. It costs nothing when the app is never recorded. +### Chat UIs over a flow in chat mode + +A flow deployed with chat mode on is a chat backend (streaming answer, tool calls, memory, conversation history). Do not drive it through a runnable: add \`windmill-chat\` to \`package.json\` and use it directly, it detects the app's Windmill and credential. + +\`\`\`tsx +import { useWindmillChat } from 'windmill-chat/react' + +const chat = useWindmillChat({ flowPath: 'f/support/assistant' }) +// chat.messages ({ role, content, pending, success, tool? }), chat.status, chat.sendMessage(text), chat.stop() +\`\`\` + +\`windmill-chat/ai-sdk\` gives a \`ChatTransport\` for the Vercel AI SDK's \`useChat\`, \`windmill-chat/assistant-ui\` a runtime for assistant-ui. The flow must be deployed, not a draft. A sandboxed app needs \`jobs:run\` in its frontend SDK scopes, plus \`flow_conversations:write\` for the conversation sidebar; without them the chat keeps history in the browser. + ## Backend runnables Each runnable has a unique key (used to call it from the frontend) and one of four types: @@ -6035,6 +6048,7 @@ def main(user_id: str): 6. **Mark sensitive UI with \`data-wm-no-record\`** — it is what keeps that data out of a recorded demo; passwords are handled for you. 7. **Reach for \`backendAsync\` + \`waitJob\`** for long work — never a hand-written job-polling runnable. 8. **Deploy what a path runnable points at** — a path runnable aimed at a draft fails at runtime; tell the user what needs deploying. +9. **Use \`windmill-chat\` for a chat over a chat-mode flow** — never a runnable that runs the flow and polls its stream. `, "triggers": `--- name: triggers diff --git a/frontend/src/lib/components/raw_apps/sdkScopes.ts b/frontend/src/lib/components/raw_apps/sdkScopes.ts index f0ece00c3e..b0177bb660 100644 --- a/frontend/src/lib/components/raw_apps/sdkScopes.ts +++ b/frontend/src/lib/components/raw_apps/sdkScopes.ts @@ -28,6 +28,16 @@ export const FRONTEND_SDK_SCOPES: { value: string; label: string; description: s value: 'variables:read', label: 'Read variables', description: 'Read variable values the viewer can access' + }, + { + value: 'flow_conversations:read', + label: 'Read your flow chats', + description: 'List your chat conversations with flows and read their messages' + }, + { + value: 'flow_conversations:write', + label: 'Manage your flow chats', + description: 'Read and delete your chat conversations with flows' } ] @@ -72,5 +82,5 @@ export function storeSdkConsent( ): void { try { localStorage.setItem(sdkConsentKey(viewer, workspace, path), JSON.stringify(scopes)) - } catch (_) { } + } catch (_) {} } diff --git a/frontend/src/lib/mcpEndpointTools.ts b/frontend/src/lib/mcpEndpointTools.ts index 1398e8dfa9..a9e2b9f3b2 100644 --- a/frontend/src/lib/mcpEndpointTools.ts +++ b/frontend/src/lib/mcpEndpointTools.ts @@ -1275,10 +1275,12 @@ export const mcpEndpointTools: EndpointTool[] = [ "description": "Who may open the app, and who its runnables execute as. Optional, and what omitting it means depends on the operation: creating an app defaults it to `publisher` (runs on behalf of the app's publisher and requires an authenticated viewer), while updating one keeps the mode the app is already deployed under. Neither `anonymous`, which makes the app publicly executable, nor `guest`, which opens it to anyone the identity provider authenticates, is ever assumed. A guest is only admitted where the workspace also has `guest_access_enabled`, which is checked when the session is minted and again on every guest request. Possible values: viewer, publisher, guest, anonymous" }, "on_behalf_of": { - "type": "string" + "type": "string", + "description": "The user or group the app runs as in anonymous or publisher mode (e.g. 'u/admin' or 'g/mygroup'). The authority for the app's identity." }, "on_behalf_of_email": { - "type": "string" + "type": "string", + "description": "Address of `on_behalf_of`, written through from it on every save and returned as stored. Optional; when absent it is derived from `on_behalf_of`. Sending it is optional too; it must name the same account as `on_behalf_of`, and a pair that disagrees is rejected." }, "sandbox": { "type": "boolean", @@ -1289,7 +1291,7 @@ export const mcpEndpointTools: EndpointTool[] = [ "items": { "type": "string" }, - "description": "Raw apps: author-declared scopes for the frontend SDK token. Takes effect only when `sandbox` is also true \u2014 an unsandboxed bundle runs with the viewer's own session, so no token is advertised or minted for it and this list stays inert. On a sandboxed app a non-empty list lets viewers mint (after consenting) a short-lived token carrying their own identity restricted to these scopes, handed to the app bundle so `windmill-client` calls run as the viewer. Must be a subset of the server's curated allowlist (jobs:run, jobs:read, users:read, resources:read, variables:read).\n" + "description": "Raw apps: author-declared scopes for the frontend SDK token. Takes effect only when `sandbox` is also true \u2014 an unsandboxed bundle runs with the viewer's own session, so no token is advertised or minted for it and this list stays inert. On a sandboxed app a non-empty list lets viewers mint (after consenting) a short-lived token carrying their own identity restricted to these scopes, handed to the app bundle so `windmill-client` calls run as the viewer. Must be a subset of the server's curated allowlist (jobs:run, jobs:read, users:read, resources:read, variables:read, flow_conversations:read, flow_conversations:write).\n" } } } @@ -1390,10 +1392,12 @@ export const mcpEndpointTools: EndpointTool[] = [ "description": "Who may open the app, and who its runnables execute as. Optional, and what omitting it means depends on the operation: creating an app defaults it to `publisher` (runs on behalf of the app's publisher and requires an authenticated viewer), while updating one keeps the mode the app is already deployed under. Neither `anonymous`, which makes the app publicly executable, nor `guest`, which opens it to anyone the identity provider authenticates, is ever assumed. A guest is only admitted where the workspace also has `guest_access_enabled`, which is checked when the session is minted and again on every guest request. Possible values: viewer, publisher, guest, anonymous" }, "on_behalf_of": { - "type": "string" + "type": "string", + "description": "The user or group the app runs as in anonymous or publisher mode (e.g. 'u/admin' or 'g/mygroup'). The authority for the app's identity." }, "on_behalf_of_email": { - "type": "string" + "type": "string", + "description": "Address of `on_behalf_of`, written through from it on every save and returned as stored. Optional; when absent it is derived from `on_behalf_of`. Sending it is optional too; it must name the same account as `on_behalf_of`, and a pair that disagrees is rejected." }, "sandbox": { "type": "boolean", @@ -1404,7 +1408,7 @@ export const mcpEndpointTools: EndpointTool[] = [ "items": { "type": "string" }, - "description": "Raw apps: author-declared scopes for the frontend SDK token. Takes effect only when `sandbox` is also true \u2014 an unsandboxed bundle runs with the viewer's own session, so no token is advertised or minted for it and this list stays inert. On a sandboxed app a non-empty list lets viewers mint (after consenting) a short-lived token carrying their own identity restricted to these scopes, handed to the app bundle so `windmill-client` calls run as the viewer. Must be a subset of the server's curated allowlist (jobs:run, jobs:read, users:read, resources:read, variables:read).\n" + "description": "Raw apps: author-declared scopes for the frontend SDK token. Takes effect only when `sandbox` is also true \u2014 an unsandboxed bundle runs with the viewer's own session, so no token is advertised or minted for it and this list stays inert. On a sandboxed app a non-empty list lets viewers mint (after consenting) a short-lived token carrying their own identity restricted to these scopes, handed to the app bundle so `windmill-client` calls run as the viewer. Must be a subset of the server's curated allowlist (jobs:run, jobs:read, users:read, resources:read, variables:read, flow_conversations:read, flow_conversations:write).\n" } } }, diff --git a/system_prompts/auto-generated/prompts.ts b/system_prompts/auto-generated/prompts.ts index 216c72c128..3529c99041 100644 --- a/system_prompts/auto-generated/prompts.ts +++ b/system_prompts/auto-generated/prompts.ts @@ -815,6 +815,19 @@ An app can be demoed by recording a session: every interaction becomes a step ca Apply it to customer data, internal notes and anything else a viewer of the demo should not see. It costs nothing when the app is never recorded. +### Chat UIs over a flow in chat mode + +A flow deployed with chat mode on is a chat backend (streaming answer, tool calls, memory, conversation history). Do not drive it through a runnable: add \`windmill-chat\` to \`package.json\` and use it directly, it detects the app's Windmill and credential. + +\`\`\`tsx +import { useWindmillChat } from 'windmill-chat/react' + +const chat = useWindmillChat({ flowPath: 'f/support/assistant' }) +// chat.messages ({ role, content, pending, success, tool? }), chat.status, chat.sendMessage(text), chat.stop() +\`\`\` + +\`windmill-chat/ai-sdk\` gives a \`ChatTransport\` for the Vercel AI SDK's \`useChat\`, \`windmill-chat/assistant-ui\` a runtime for assistant-ui. The flow must be deployed, not a draft. A sandboxed app needs \`jobs:run\` in its frontend SDK scopes, plus \`flow_conversations:write\` for the conversation sidebar; without them the chat keeps history in the browser. + ## Backend runnables Each runnable has a unique key (used to call it from the frontend) and one of four types: @@ -943,6 +956,7 @@ def main(user_id: str): 6. **Mark sensitive UI with \`data-wm-no-record\`** — it is what keeps that data out of a recorded demo; passwords are handled for you. 7. **Reach for \`backendAsync\` + \`waitJob\`** for long work — never a hand-written job-polling runnable. 8. **Deploy what a path runnable points at** — a path runnable aimed at a draft fails at runtime; tell the user what needs deploying. +9. **Use \`windmill-chat\` for a chat over a chat-mode flow** — never a runnable that runs the flow and polls its stream. `; export const PIPELINE_BASE = `# Data pipeline authoring diff --git a/system_prompts/auto-generated/skills/raw-app/SKILL.md b/system_prompts/auto-generated/skills/raw-app/SKILL.md index 05258118af..da4a9729be 100644 --- a/system_prompts/auto-generated/skills/raw-app/SKILL.md +++ b/system_prompts/auto-generated/skills/raw-app/SKILL.md @@ -330,6 +330,19 @@ An app can be demoed by recording a session: every interaction becomes a step ca Apply it to customer data, internal notes and anything else a viewer of the demo should not see. It costs nothing when the app is never recorded. +### Chat UIs over a flow in chat mode + +A flow deployed with chat mode on is a chat backend (streaming answer, tool calls, memory, conversation history). Do not drive it through a runnable: add `windmill-chat` to `package.json` and use it directly, it detects the app's Windmill and credential. + +```tsx +import { useWindmillChat } from 'windmill-chat/react' + +const chat = useWindmillChat({ flowPath: 'f/support/assistant' }) +// chat.messages ({ role, content, pending, success, tool? }), chat.status, chat.sendMessage(text), chat.stop() +``` + +`windmill-chat/ai-sdk` gives a `ChatTransport` for the Vercel AI SDK's `useChat`, `windmill-chat/assistant-ui` a runtime for assistant-ui. The flow must be deployed, not a draft. A sandboxed app needs `jobs:run` in its frontend SDK scopes, plus `flow_conversations:write` for the conversation sidebar; without them the chat keeps history in the browser. + ## Backend runnables Each runnable has a unique key (used to call it from the frontend) and one of four types: @@ -458,3 +471,4 @@ def main(user_id: str): 6. **Mark sensitive UI with `data-wm-no-record`** — it is what keeps that data out of a recorded demo; passwords are handled for you. 7. **Reach for `backendAsync` + `waitJob`** for long work — never a hand-written job-polling runnable. 8. **Deploy what a path runnable points at** — a path runnable aimed at a draft fails at runtime; tell the user what needs deploying. +9. **Use `windmill-chat` for a chat over a chat-mode flow** — never a runnable that runs the flow and polls its stream. diff --git a/system_prompts/base/raw-app.md b/system_prompts/base/raw-app.md index fde0b835ca..3849a465a0 100644 --- a/system_prompts/base/raw-app.md +++ b/system_prompts/base/raw-app.md @@ -95,6 +95,19 @@ An app can be demoed by recording a session: every interaction becomes a step ca Apply it to customer data, internal notes and anything else a viewer of the demo should not see. It costs nothing when the app is never recorded. +### Chat UIs over a flow in chat mode + +A flow deployed with chat mode on is a chat backend (streaming answer, tool calls, memory, conversation history). Do not drive it through a runnable: add `windmill-chat` to `package.json` and use it directly, it detects the app's Windmill and credential. + +```tsx +import { useWindmillChat } from 'windmill-chat/react' + +const chat = useWindmillChat({ flowPath: 'f/support/assistant' }) +// chat.messages ({ role, content, pending, success, tool? }), chat.status, chat.sendMessage(text), chat.stop() +``` + +`windmill-chat/ai-sdk` gives a `ChatTransport` for the Vercel AI SDK's `useChat`, `windmill-chat/assistant-ui` a runtime for assistant-ui. The flow must be deployed, not a draft. A sandboxed app needs `jobs:run` in its frontend SDK scopes, plus `flow_conversations:write` for the conversation sidebar; without them the chat keeps history in the browser. + ## Backend runnables Each runnable has a unique key (used to call it from the frontend) and one of four types: @@ -223,3 +236,4 @@ def main(user_id: str): 6. **Mark sensitive UI with `data-wm-no-record`** — it is what keeps that data out of a recorded demo; passwords are handled for you. 7. **Reach for `backendAsync` + `waitJob`** for long work — never a hand-written job-polling runnable. 8. **Deploy what a path runnable points at** — a path runnable aimed at a draft fails at runtime; tell the user what needs deploying. +9. **Use `windmill-chat` for a chat over a chat-mode flow** — never a runnable that runs the flow and polls its stream. From 57a134e2de18e27b3c1d3066a60b892af1089b30 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 14 Sep 2026 22:46:26 +0200 Subject: [PATCH 13/44] feat(ai-sessions): share session artifacts with the workspace by link (#11115) * feat(ai-sessions): share session artifacts with the workspace by link Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Pyjp67oR269QAx3b4yf4oH * chore: cache the shared artifact queries for offline sqlx Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Pyjp67oR269QAx3b4yf4oH * fix: replace a literal NUL byte in the shared artifact body limit comment Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Pyjp67oR269QAx3b4yf4oH * test: pin that a shared artifact is confined to its workspace's path Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01Pyjp67oR269QAx3b4yf4oH * fix: sanitize shared artifact markdown and validate the artifact id on every route The shared page renders another member's markdown, so ArtifactBody now runs the repo's rehype-raw + rehype-sanitize chain with the chat's link renderer on top; only the session viewer opts into the chat code block (mermaid, apply button). The link renderer keeps a link's text when its href is empty or unsafe, and the scheme check moves to a tested helper. The status route checks artifact_id like share does, so a NUL is a 400 rather than a 500. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01Pyjp67oR269QAx3b4yf4oH * fix(ai-sessions): say which way re-sharing moves an artifact link The popover offered "Update to v1" when a v2 link was open on a pinned v1, which reads as if v1 were newer. Each direction now has its own sentence and action: a newer version on screen updates the link, an older one shares that version instead, a rename updates the name. Co-Authored-By: Claude Fable 5.1 --------- Co-authored-by: Claude Opus 5 (1M context) --- ...a6cf0148a9e10c1c4ad0f99312b8e99ec1b8a.json | 41 +++ ...74b8c5beb1c3992f6413d7bcabc8159c0f9bb.json | 14 + ...9207306fb19756ab9c3a3d7824a15542a5b42.json | 66 ++++ ...804306fe2083928e57458ba936d6afde53b57.json | 55 +++ ...107739baa9ee276a99f481335fec8099f4d51.json | 25 ++ ...20260914135805_ai_shared_artifact.down.sql | 1 + .../20260914135805_ai_shared_artifact.up.sql | 32 ++ backend/src/monitor.rs | 11 + backend/summarized_schema.txt | 2 + backend/tests/ai_shared_artifacts.rs | 184 ++++++++++ .../tests/fixtures/ai_shared_artifacts.sql | 17 + backend/windmill-api/openapi.yaml | 158 +++++++++ backend/windmill-api/src/ai.rs | 4 + .../windmill-api/src/ai_shared_artifacts.rs | 322 ++++++++++++++++++ backend/windmill-api/src/lib.rs | 1 + .../windmill-common/src/global_settings.rs | 1 + backend/windmill-common/src/lib.rs | 17 + .../copilot/chat/LinkRenderer.svelte | 12 +- .../chat/artifacts/ArtifactBody.svelte | 46 +++ .../artifacts/ArtifactExportButton.svelte | 42 +++ .../chat/artifacts/ArtifactShareButton.svelte | 196 +++++++++++ .../chat/artifacts/ArtifactViewer.svelte | 83 ++--- .../chat/artifacts/artifactSharing.test.ts | 21 ++ .../copilot/chat/artifacts/artifactSharing.ts | 42 +++ .../components/copilot/chat/safeHref.test.ts | 33 ++ .../lib/components/copilot/chat/safeHref.ts | 17 + .../shared_artifacts/[id]/+page.svelte | 144 ++++++++ .../(logged)/shared_artifacts/[id]/+page.ts | 5 + 28 files changed, 1527 insertions(+), 65 deletions(-) create mode 100644 backend/.sqlx/query-0589cb0f96e17ecadae4923be70a6cf0148a9e10c1c4ad0f99312b8e99ec1b8a.json create mode 100644 backend/.sqlx/query-6774bc0ec8ca8c6c48e8e111ab074b8c5beb1c3992f6413d7bcabc8159c0f9bb.json create mode 100644 backend/.sqlx/query-945230149990abda67fdf4779529207306fb19756ab9c3a3d7824a15542a5b42.json create mode 100644 backend/.sqlx/query-d2861932a739887785658cdf89a804306fe2083928e57458ba936d6afde53b57.json create mode 100644 backend/.sqlx/query-e50660f58274e9c135ace356ea8107739baa9ee276a99f481335fec8099f4d51.json create mode 100644 backend/migrations/20260914135805_ai_shared_artifact.down.sql create mode 100644 backend/migrations/20260914135805_ai_shared_artifact.up.sql create mode 100644 backend/tests/ai_shared_artifacts.rs create mode 100644 backend/tests/fixtures/ai_shared_artifacts.sql create mode 100644 backend/windmill-api/src/ai_shared_artifacts.rs create mode 100644 frontend/src/lib/components/copilot/chat/artifacts/ArtifactBody.svelte create mode 100644 frontend/src/lib/components/copilot/chat/artifacts/ArtifactExportButton.svelte create mode 100644 frontend/src/lib/components/copilot/chat/artifacts/ArtifactShareButton.svelte create mode 100644 frontend/src/lib/components/copilot/chat/artifacts/artifactSharing.test.ts create mode 100644 frontend/src/lib/components/copilot/chat/artifacts/artifactSharing.ts create mode 100644 frontend/src/lib/components/copilot/chat/safeHref.test.ts create mode 100644 frontend/src/lib/components/copilot/chat/safeHref.ts create mode 100644 frontend/src/routes/(root)/(logged)/shared_artifacts/[id]/+page.svelte create mode 100644 frontend/src/routes/(root)/(logged)/shared_artifacts/[id]/+page.ts diff --git a/backend/.sqlx/query-0589cb0f96e17ecadae4923be70a6cf0148a9e10c1c4ad0f99312b8e99ec1b8a.json b/backend/.sqlx/query-0589cb0f96e17ecadae4923be70a6cf0148a9e10c1c4ad0f99312b8e99ec1b8a.json new file mode 100644 index 0000000000..6558ea16e5 --- /dev/null +++ b/backend/.sqlx/query-0589cb0f96e17ecadae4923be70a6cf0148a9e10c1c4ad0f99312b8e99ec1b8a.json @@ -0,0 +1,41 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO ai_shared_artifact\n (workspace_id, artifact_id, email, created_by, name, kind, version, content)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8)\n ON CONFLICT (workspace_id, email, artifact_id) DO UPDATE\n SET created_by = EXCLUDED.created_by,\n name = EXCLUDED.name,\n kind = EXCLUDED.kind,\n version = EXCLUDED.version,\n content = EXCLUDED.content,\n shared_at = now()\n RETURNING id, shared_at, (xmax = 0) AS \"inserted!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "shared_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 2, + "name": "inserted!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Int4", + "Text" + ] + }, + "nullable": [ + false, + false, + null + ] + }, + "hash": "0589cb0f96e17ecadae4923be70a6cf0148a9e10c1c4ad0f99312b8e99ec1b8a" +} diff --git a/backend/.sqlx/query-6774bc0ec8ca8c6c48e8e111ab074b8c5beb1c3992f6413d7bcabc8159c0f9bb.json b/backend/.sqlx/query-6774bc0ec8ca8c6c48e8e111ab074b8c5beb1c3992f6413d7bcabc8159c0f9bb.json new file mode 100644 index 0000000000..8dd7954460 --- /dev/null +++ b/backend/.sqlx/query-6774bc0ec8ca8c6c48e8e111ab074b8c5beb1c3992f6413d7bcabc8159c0f9bb.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM ai_shared_artifact\n WHERE shared_at <= now() - ($1::bigint::text || ' s')::interval", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "6774bc0ec8ca8c6c48e8e111ab074b8c5beb1c3992f6413d7bcabc8159c0f9bb" +} diff --git a/backend/.sqlx/query-945230149990abda67fdf4779529207306fb19756ab9c3a3d7824a15542a5b42.json b/backend/.sqlx/query-945230149990abda67fdf4779529207306fb19756ab9c3a3d7824a15542a5b42.json new file mode 100644 index 0000000000..542d523f53 --- /dev/null +++ b/backend/.sqlx/query-945230149990abda67fdf4779529207306fb19756ab9c3a3d7824a15542a5b42.json @@ -0,0 +1,66 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, email, name, kind, version, created_by, content, shared_at\n FROM ai_shared_artifact\n WHERE workspace_id = $1 AND id = $2\n AND shared_at > now() - ($3::bigint::text || ' s')::interval", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "kind", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "version", + "type_info": "Int4" + }, + { + "ordinal": 5, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 6, + "name": "content", + "type_info": "Text" + }, + { + "ordinal": 7, + "name": "shared_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Uuid", + "Int8" + ] + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false + ] + }, + "hash": "945230149990abda67fdf4779529207306fb19756ab9c3a3d7824a15542a5b42" +} diff --git a/backend/.sqlx/query-d2861932a739887785658cdf89a804306fe2083928e57458ba936d6afde53b57.json b/backend/.sqlx/query-d2861932a739887785658cdf89a804306fe2083928e57458ba936d6afde53b57.json new file mode 100644 index 0000000000..1e56d65715 --- /dev/null +++ b/backend/.sqlx/query-d2861932a739887785658cdf89a804306fe2083928e57458ba936d6afde53b57.json @@ -0,0 +1,55 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, name, kind, version, created_by, shared_at FROM ai_shared_artifact\n WHERE workspace_id = $1 AND email = $2 AND artifact_id = $3\n AND shared_at > now() - ($4::bigint::text || ' s')::interval", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "kind", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "version", + "type_info": "Int4" + }, + { + "ordinal": 4, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "shared_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Int8" + ] + }, + "nullable": [ + false, + false, + false, + false, + false, + false + ] + }, + "hash": "d2861932a739887785658cdf89a804306fe2083928e57458ba936d6afde53b57" +} diff --git a/backend/.sqlx/query-e50660f58274e9c135ace356ea8107739baa9ee276a99f481335fec8099f4d51.json b/backend/.sqlx/query-e50660f58274e9c135ace356ea8107739baa9ee276a99f481335fec8099f4d51.json new file mode 100644 index 0000000000..3068e4fe35 --- /dev/null +++ b/backend/.sqlx/query-e50660f58274e9c135ace356ea8107739baa9ee276a99f481335fec8099f4d51.json @@ -0,0 +1,25 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM ai_shared_artifact\n WHERE workspace_id = $1 AND id = $2 AND (email = $3 OR $4::bool)\n RETURNING name", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Uuid", + "Text", + "Bool" + ] + }, + "nullable": [ + false + ] + }, + "hash": "e50660f58274e9c135ace356ea8107739baa9ee276a99f481335fec8099f4d51" +} diff --git a/backend/migrations/20260914135805_ai_shared_artifact.down.sql b/backend/migrations/20260914135805_ai_shared_artifact.down.sql new file mode 100644 index 0000000000..04fa92db6c --- /dev/null +++ b/backend/migrations/20260914135805_ai_shared_artifact.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS ai_shared_artifact; diff --git a/backend/migrations/20260914135805_ai_shared_artifact.up.sql b/backend/migrations/20260914135805_ai_shared_artifact.up.sql new file mode 100644 index 0000000000..6daa3e5daf --- /dev/null +++ b/backend/migrations/20260914135805_ai_shared_artifact.up.sql @@ -0,0 +1,32 @@ +-- A copy of an AI session artifact that its author explicitly shared with the workspace. +-- Artifacts otherwise live only in the author's browser; this row exists only while the +-- share does, and the monitor deletes it once `shared_at` falls outside +-- AI_SHARED_ARTIFACT_RETENTION_SECS. +CREATE TABLE ai_shared_artifact ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE, + -- The browser-side artifact id. Unique per author so sharing the same artifact again + -- moves its one link forward rather than minting a second one. + artifact_id VARCHAR(255) NOT NULL, + email VARCHAR(255) NOT NULL, + created_by VARCHAR(255) NOT NULL, + name VARCHAR(255) NOT NULL, + kind VARCHAR(10) NOT NULL CHECK (kind IN ('md', 'html')), + version INTEGER NOT NULL, + content TEXT NOT NULL, + -- Reset on every re-share: retention counts from the last time the author shared it. + shared_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE (workspace_id, email, artifact_id) +); + +CREATE INDEX idx_ai_shared_artifact_shared_at ON ai_shared_artifact (shared_at); + +GRANT ALL ON ai_shared_artifact TO windmill_admin; +GRANT ALL ON ai_shared_artifact TO windmill_user; + +-- The handlers go through the raw pool and scope every query to the workspace themselves. +-- An admin-only policy is the backstop for a future query that reaches this table through +-- UserDB. +ALTER TABLE ai_shared_artifact ENABLE ROW LEVEL SECURITY; + +CREATE POLICY admin_policy ON ai_shared_artifact FOR ALL TO windmill_admin USING (true); diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index d18b23e1c0..493abd172e 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -1912,6 +1912,17 @@ pub async fn delete_expired_items(db: &DB) -> () { tracing::info!("deleted {} expired otel trace spans", deleted_spans); } + if let Err(e) = sqlx::query!( + "DELETE FROM ai_shared_artifact + WHERE shared_at <= now() - ($1::bigint::text || ' s')::interval", + windmill_common::ai_shared_artifact_retention_secs(), + ) + .execute(db) + .await + { + tracing::error!("Error deleting expired shared AI artifacts: {:?}", e); + } + let audit_retention_days = audit_log_retention_days().await; let audit_retention_secs: i64 = audit_retention_days * 60 * 60 * 24; diff --git a/backend/summarized_schema.txt b/backend/summarized_schema.txt index 41dd70ca93..1ab05bc584 100644 --- a/backend/summarized_schema.txt +++ b/backend/summarized_schema.txt @@ -40,6 +40,8 @@ agent_token_blacklist: token(char), expires_at(ts), blacklisted_at(ts), blacklis ai_agent_memory: workspace_id(char), conversation_id(uuid), step_id(char), messages(jsonb), created_at(ts), updated_at(ts) ai_free_token_daily_usage: day(date), cost_nanos(bigint), updated_at(ts) ai_free_token_usage: email(char), cost_nanos(bigint), updated_at(ts) +ai_shared_artifact: id(uuid), workspace_id(char), artifact_id(char), email(char), created_by(char), name(char), kind(char), version(int), content(text), shared_at(ts) + FK: (workspace_id) -> workspace(id) ai_token_usage: workspace_id(char), day(date), email(char), provider(char), model(char), session_id(char), input_tokens(bigint), cache_read_tokens(bigint), cache_write_tokens(bigint), output_tokens(bigint), reported_cost_nano_usd(bigint), requests(bigint), updated_at(ts) FK: (workspace_id) -> workspace(id) alerts: id(int), alert_type(char), message(text), created_at(ts), acknowledged(bool), workspace_id(text), acknowledged_workspace(bool), resource(text) diff --git a/backend/tests/ai_shared_artifacts.rs b/backend/tests/ai_shared_artifacts.rs new file mode 100644 index 0000000000..17dd313755 --- /dev/null +++ b/backend/tests/ai_shared_artifacts.rs @@ -0,0 +1,184 @@ +//! Shared AI session artifacts: one link per author and artifact, readable by any workspace +//! member until its retention window passes, and removable only by its author or an admin. +//! +//! Expiry is enforced on read as well as by the monitor's sweep, so a share past its window must +//! not be served in the gap before the sweep reaches it. + +use serde_json::{json, Value}; +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +const ADMIN: &str = "Bearer SECRET_TOKEN"; +const MEMBER: &str = "Bearer SECRET_TOKEN_2"; + +async fn share( + client: &reqwest::Client, + base: &str, + token: &str, + content: &str, +) -> anyhow::Result { + let resp = client + .post(format!("{base}/share")) + .header("Authorization", token) + .json(&json!({ + "artifact_id": "plan:session-1", + "name": "Plan", + "kind": "md", + "version": 1, + "content": content, + })) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + Ok(resp.json().await?) +} + +#[sqlx::test(fixtures("base", "ai_shared_artifacts"))] +async fn shared_artifact_is_served_to_members_until_it_expires( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let base = format!( + "http://localhost:{}/api/w/test-workspace/ai/shared_artifacts", + server.addr.port() + ); + let client = reqwest::Client::new(); + + let first = share(&client, &base, MEMBER, "draft").await?; + let second = share(&client, &base, MEMBER, "final").await?; + assert_eq!(first["id"], second["id"], "re-sharing minted a second link"); + let id = second["id"].as_str().unwrap(); + + let resp = client + .get(format!("{base}/get/{id}")) + .header("Authorization", ADMIN) + .send() + .await?; + assert_eq!(resp.status(), 200); + let body: Value = resp.json().await?; + assert_eq!(body["content"], "final"); + + // The handlers read through the raw pool, so the workspace in the URL is the only thing + // scoping a share: a member of another workspace must not reach it by id through theirs. + let resp = client + .get(format!( + "http://localhost:{}/api/w/test-workspace-2/ai/shared_artifacts/get/{id}", + server.addr.port() + )) + .header("Authorization", ADMIN) + .send() + .await?; + assert_eq!( + resp.status(), + 404, + "a share was served through another workspace's path" + ); + + sqlx::query( + "UPDATE ai_shared_artifact SET shared_at = now() - ($1::bigint + 60) * interval '1 second'", + ) + .bind(windmill_common::ai_shared_artifact_retention_secs()) + .execute(&db) + .await?; + + let resp = client + .get(format!("{base}/get/{id}")) + .header("Authorization", ADMIN) + .send() + .await?; + assert_eq!(resp.status(), 404, "an expired share was served"); + + let status: Value = client + .get(format!("{base}/status?artifact_id=plan:session-1")) + .header("Authorization", MEMBER) + .send() + .await? + .json() + .await?; + assert!( + status.get("share").is_none(), + "an expired share was reported live: {status}" + ); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn only_the_author_or_an_admin_can_unshare(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let base = format!( + "http://localhost:{}/api/w/test-workspace/ai/shared_artifacts", + server.addr.port() + ); + let client = reqwest::Client::new(); + + let shared = share(&client, &base, ADMIN, "admin's plan").await?; + let id = shared["id"].as_str().unwrap(); + + let resp = client + .delete(format!("{base}/delete/{id}")) + .header("Authorization", MEMBER) + .send() + .await?; + assert_eq!(resp.status(), 404); + let remaining: i64 = sqlx::query_scalar("SELECT count(*) FROM ai_shared_artifact") + .fetch_one(&db) + .await?; + assert_eq!(remaining, 1, "a member deleted someone else's share"); + + let member_share = share(&client, &base, MEMBER, "member's plan").await?; + let resp = client + .delete(format!( + "{base}/delete/{}", + member_share["id"].as_str().unwrap() + )) + .header("Authorization", ADMIN) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + + Ok(()) +} + +/// The id is compared against a `VARCHAR(255)` column on every route that takes one, and a +/// NUL in it would otherwise reach Postgres and come back as a 500. +#[sqlx::test(fixtures("base"))] +async fn a_malformed_artifact_id_is_refused_on_every_route( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let base = format!( + "http://localhost:{}/api/w/test-workspace/ai/shared_artifacts", + server.addr.port() + ); + let client = reqwest::Client::new(); + + for bad_id in ["", "a\0b", &"x".repeat(256)] { + let resp = client + .get(format!("{base}/status")) + .query(&[("artifact_id", bad_id)]) + .header("Authorization", MEMBER) + .send() + .await?; + assert_eq!(resp.status(), 400, "status accepted {bad_id:?}"); + + let resp = client + .post(format!("{base}/share")) + .header("Authorization", MEMBER) + .json(&json!({ + "artifact_id": bad_id, + "name": "Plan", + "kind": "md", + "version": 1, + "content": "x", + })) + .send() + .await?; + assert_eq!(resp.status(), 400, "share accepted {bad_id:?}"); + } + + Ok(()) +} diff --git a/backend/tests/fixtures/ai_shared_artifacts.sql b/backend/tests/fixtures/ai_shared_artifacts.sql new file mode 100644 index 0000000000..5f745ed0cf --- /dev/null +++ b/backend/tests/fixtures/ai_shared_artifacts.sql @@ -0,0 +1,17 @@ +-- Layers on `base`: a second workspace the superadmin `test-user` is also a member of, so a +-- share can be requested through the wrong workspace's path by a caller the route accepts. + +INSERT INTO workspace (id, name, owner) VALUES + ('test-workspace-2', 'test-workspace-2', 'test-user'); + +INSERT INTO workspace_key(workspace_id, kind, key) VALUES + ('test-workspace-2', 'cloud', 'test-key-2'); + +INSERT INTO workspace_settings (workspace_id) VALUES + ('test-workspace-2'); + +INSERT INTO group_ (workspace_id, name, summary, extra_perms) VALUES + ('test-workspace-2', 'all', 'All users', '{}'); + +INSERT INTO usr(workspace_id, email, username, is_admin, role) VALUES + ('test-workspace-2', 'test@windmill.dev', 'test-user', true, 'Admin'); diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index aee198e92a..411d5f658c 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -13191,6 +13191,134 @@ paths: type: boolean description: more buckets matched than were returned, so summing them under-reports + /w/{workspace}/ai/shared_artifacts/share: + post: + summary: share an AI session artifact with the workspace + description: > + Stores a read-only copy that any member of the workspace can open by id. Sharing the + same artifact again updates that copy, keeps its id, and restarts its retention window. + operationId: shareAiArtifact + tags: + - ai + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - artifact_id + - name + - kind + - version + - content + properties: + artifact_id: + type: string + description: the artifact's id in the author's session + name: + type: string + kind: + type: string + enum: [md, html] + version: + type: integer + minimum: 1 + content: + type: string + responses: + "200": + description: the shared copy + content: + application/json: + schema: + $ref: "#/components/schemas/SharedAiArtifactInfo" + + /w/{workspace}/ai/shared_artifacts/status: + get: + summary: get the calling user's share of one of their AI session artifacts + operationId: getAiArtifactShareStatus + tags: + - ai + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: artifact_id + in: query + required: true + schema: + type: string + responses: + "200": + description: the live share, if any, and how long shares last + content: + application/json: + schema: + type: object + required: + - retention_secs + properties: + retention_secs: + type: integer + share: + $ref: "#/components/schemas/SharedAiArtifactInfo" + + /w/{workspace}/ai/shared_artifacts/get/{id}: + get: + summary: get a shared AI session artifact + operationId: getSharedAiArtifact + tags: + - ai + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: id + in: path + required: true + schema: + type: string + format: uuid + responses: + "200": + description: the shared artifact + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/SharedAiArtifactInfo" + - type: object + required: + - content + - can_unshare + properties: + content: + type: string + can_unshare: + type: boolean + description: whether the caller authored the share or is a workspace admin + + /w/{workspace}/ai/shared_artifacts/delete/{id}: + delete: + summary: stop sharing an AI session artifact + operationId: unshareAiArtifact + tags: + - ai + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: id + in: path + required: true + schema: + type: string + format: uuid + responses: + "200": + description: share deleted + content: + text/plain: + schema: + type: string + /w/{workspace}/apps/get_data/v/{secretWithExtension}: get: summary: get raw app data by @@ -28256,6 +28384,36 @@ components: - output_tokens - requests + SharedAiArtifactInfo: + type: object + properties: + id: + type: string + format: uuid + name: + type: string + kind: + type: string + enum: [md, html] + version: + type: integer + created_by: + type: string + shared_at: + type: string + format: date-time + expires_at: + type: string + format: date-time + required: + - id + - name + - kind + - version + - created_by + - shared_at + - expires_at + InstanceAIProviderSummary: type: object properties: diff --git a/backend/windmill-api/src/ai.rs b/backend/windmill-api/src/ai.rs index e70cc19544..8101064c3d 100644 --- a/backend/windmill-api/src/ai.rs +++ b/backend/windmill-api/src/ai.rs @@ -518,6 +518,10 @@ pub fn workspaced_service() -> Router { // could make the server allocate and parse an arbitrarily large one. // Sized well above a full batch of the shape below. .layer(DefaultBodyLimit::max(AI_USAGE_BODY_LIMIT)), + ) + .nest( + "/shared_artifacts", + crate::ai_shared_artifacts::workspaced_service(), ); #[cfg(feature = "bedrock")] diff --git a/backend/windmill-api/src/ai_shared_artifacts.rs b/backend/windmill-api/src/ai_shared_artifacts.rs new file mode 100644 index 0000000000..36ad0ecd2e --- /dev/null +++ b/backend/windmill-api/src/ai_shared_artifacts.rs @@ -0,0 +1,322 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2026 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Read-only copies of AI session artifacts, shared with the workspace by their author. +//! +//! Artifacts live in the author's browser; a row here exists only because the author asked +//! for a link. Every read filters on the retention window as well as the monitor sweeping +//! it, so a share past its window is never served in the gap before the sweep reaches it. + +use crate::db::{ApiAuthed, DB}; +use axum::{ + extract::{DefaultBodyLimit, Extension, Json, Path, Query}, + routing::{delete, get, post}, + Router, +}; +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; +use windmill_audit::{audit_oss::audit_log, ActionKind}; +use windmill_common::{ + ai_shared_artifact_retention_secs, + error::{Error, JsonResult, Result}, +}; + +/// The frontend's `MAX_ARTIFACT_BYTES`: no artifact the viewer holds is larger. +const MAX_CONTENT_BYTES: usize = 256 * 1024; +const MAX_NAME_CHARS: usize = 255; +const MAX_ARTIFACT_ID_CHARS: usize = 255; +/// JSON escapes a control character into six bytes, so a full-size artifact made entirely of +/// them still fits. +const SHARE_BODY_LIMIT: usize = MAX_CONTENT_BYTES * 6 + 64 * 1024; + +pub fn workspaced_service() -> Router { + Router::new() + .route( + "/share", + post(share_artifact).layer(DefaultBodyLimit::max(SHARE_BODY_LIMIT)), + ) + .route("/status", get(get_share_status)) + .route("/get/{id}", get(get_shared_artifact)) + .route("/delete/{id}", delete(unshare_artifact)) +} + +#[derive(Deserialize, Clone, Copy)] +#[serde(rename_all = "lowercase")] +enum ArtifactKind { + Md, + Html, +} + +impl ArtifactKind { + fn as_str(self) -> &'static str { + match self { + ArtifactKind::Md => "md", + ArtifactKind::Html => "html", + } + } +} + +#[derive(Deserialize)] +struct ShareArtifact { + artifact_id: String, + name: String, + kind: ArtifactKind, + version: i32, + content: String, +} + +#[derive(Serialize)] +struct SharedArtifactInfo { + id: Uuid, + name: String, + kind: String, + version: i32, + created_by: String, + shared_at: DateTime, + expires_at: DateTime, +} + +#[derive(Serialize)] +struct SharedArtifact { + #[serde(flatten)] + info: SharedArtifactInfo, + content: String, + /// Whether the caller may stop sharing it: its author, or a workspace admin. + can_unshare: bool, +} + +#[derive(Serialize)] +struct ShareStatus { + retention_secs: i64, + #[serde(skip_serializing_if = "Option::is_none")] + share: Option, +} + +#[derive(Deserialize)] +struct ShareStatusQuery { + artifact_id: String, +} + +fn expires_at(shared_at: DateTime, retention_secs: i64) -> DateTime { + shared_at + chrono::Duration::seconds(retention_secs) +} + +/// The browser-side artifact id, as every handler that takes one must check it: it is compared +/// against a `VARCHAR(255)` column, and Postgres answers a NUL in a text parameter with an +/// opaque 500. +fn check_artifact_id(artifact_id: &str) -> Result<()> { + if artifact_id.is_empty() || artifact_id.chars().count() > MAX_ARTIFACT_ID_CHARS { + return Err(Error::BadRequest(format!( + "Artifact id must be between 1 and {MAX_ARTIFACT_ID_CHARS} characters" + ))); + } + if artifact_id.contains('\0') { + return Err(Error::BadRequest( + "Artifact id cannot contain NUL characters".to_string(), + )); + } + Ok(()) +} + +/// Share an artifact, or move the caller's existing link for it to this content. Re-sharing +/// keeps the link and restarts its retention window. +async fn share_artifact( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Json(payload): Json, +) -> JsonResult { + let name = payload.name.trim(); + if name.is_empty() || name.chars().count() > MAX_NAME_CHARS { + return Err(Error::BadRequest(format!( + "Artifact name must be between 1 and {MAX_NAME_CHARS} characters" + ))); + } + check_artifact_id(&payload.artifact_id)?; + if payload.content.len() > MAX_CONTENT_BYTES { + return Err(Error::BadRequest(format!( + "Artifact content is {} bytes, above the {MAX_CONTENT_BYTES} byte limit", + payload.content.len() + ))); + } + if payload.version < 1 { + return Err(Error::BadRequest( + "Artifact version must be at least 1".to_string(), + )); + } + // Postgres rejects NUL in text columns with an opaque 500. + if name.contains('\0') || payload.content.contains('\0') { + return Err(Error::BadRequest( + "Artifact name and content cannot contain NUL characters".to_string(), + )); + } + + let mut tx = db.begin().await?; + let row = sqlx::query!( + r#"INSERT INTO ai_shared_artifact + (workspace_id, artifact_id, email, created_by, name, kind, version, content) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) + ON CONFLICT (workspace_id, email, artifact_id) DO UPDATE + SET created_by = EXCLUDED.created_by, + name = EXCLUDED.name, + kind = EXCLUDED.kind, + version = EXCLUDED.version, + content = EXCLUDED.content, + shared_at = now() + RETURNING id, shared_at, (xmax = 0) AS "inserted!""#, + &w_id, + &payload.artifact_id, + &authed.email, + &authed.username, + name, + payload.kind.as_str(), + payload.version, + &payload.content, + ) + .fetch_one(&mut *tx) + .await?; + + let id = row.id.to_string(); + audit_log( + &mut *tx, + &authed, + "ai.shared_artifacts.share", + if row.inserted { + ActionKind::Create + } else { + ActionKind::Update + }, + &w_id, + Some(&id), + Some([("name", name)].into()), + ) + .await?; + tx.commit().await?; + + Ok(Json(SharedArtifactInfo { + id: row.id, + name: name.to_string(), + kind: payload.kind.as_str().to_string(), + version: payload.version, + created_by: authed.username.clone(), + shared_at: row.shared_at, + expires_at: expires_at(row.shared_at, ai_shared_artifact_retention_secs()), + })) +} + +/// The caller's own live share of one of their artifacts, if any, and how long a share lasts. +async fn get_share_status( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Query(query): Query, +) -> JsonResult { + check_artifact_id(&query.artifact_id)?; + let retention_secs = ai_shared_artifact_retention_secs(); + let share = sqlx::query!( + "SELECT id, name, kind, version, created_by, shared_at FROM ai_shared_artifact + WHERE workspace_id = $1 AND email = $2 AND artifact_id = $3 + AND shared_at > now() - ($4::bigint::text || ' s')::interval", + &w_id, + &authed.email, + &query.artifact_id, + retention_secs, + ) + .fetch_optional(&db) + .await? + .map(|r| SharedArtifactInfo { + id: r.id, + name: r.name, + kind: r.kind, + version: r.version, + created_by: r.created_by, + shared_at: r.shared_at, + expires_at: expires_at(r.shared_at, retention_secs), + }); + + Ok(Json(ShareStatus { retention_secs, share })) +} + +/// Any member of the workspace may read a live share: that is what sharing it granted. +async fn get_shared_artifact( + authed: ApiAuthed, + Extension(db): Extension, + Path((w_id, id)): Path<(String, Uuid)>, +) -> JsonResult { + let retention_secs = ai_shared_artifact_retention_secs(); + let row = sqlx::query!( + "SELECT id, email, name, kind, version, created_by, content, shared_at + FROM ai_shared_artifact + WHERE workspace_id = $1 AND id = $2 + AND shared_at > now() - ($3::bigint::text || ' s')::interval", + &w_id, + id, + retention_secs, + ) + .fetch_optional(&db) + .await? + .ok_or_else(|| { + Error::NotFound(format!( + "Shared artifact {id} not found: it may have expired or been unshared" + )) + })?; + + Ok(Json(SharedArtifact { + can_unshare: row.email == authed.email || authed.is_admin, + content: row.content, + info: SharedArtifactInfo { + id: row.id, + name: row.name, + kind: row.kind, + version: row.version, + created_by: row.created_by, + shared_at: row.shared_at, + expires_at: expires_at(row.shared_at, retention_secs), + }, + })) +} + +async fn unshare_artifact( + authed: ApiAuthed, + Extension(db): Extension, + Path((w_id, id)): Path<(String, Uuid)>, +) -> Result { + let mut tx = db.begin().await?; + let name = sqlx::query_scalar!( + "DELETE FROM ai_shared_artifact + WHERE workspace_id = $1 AND id = $2 AND (email = $3 OR $4::bool) + RETURNING name", + &w_id, + id, + &authed.email, + authed.is_admin, + ) + .fetch_optional(&mut *tx) + .await? + .ok_or_else(|| { + Error::NotFound(format!( + "Shared artifact {id} not found, or not shared by you" + )) + })?; + + let id_str = id.to_string(); + audit_log( + &mut *tx, + &authed, + "ai.shared_artifacts.unshare", + ActionKind::Delete, + &w_id, + Some(&id_str), + Some([("name", name.as_str())].into()), + ) + .await?; + tx.commit().await?; + + Ok(format!("Stopped sharing {name}")) +} diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index bd6fda015b..fc90852ae6 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -69,6 +69,7 @@ mod ai; #[cfg(feature = "private")] mod ai_free_tier_ee; mod ai_free_tier_oss; +mod ai_shared_artifacts; mod apps; mod apps_raw_bundle; pub use apps::invalidate_app_policy_cache; diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index c0ed63cd53..49efa26f60 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -577,6 +577,7 @@ pub const ENV_SETTINGS: &[&str] = &[ "OTEL_RESOURCE_ATTRIBUTES", "OTEL_JOB_LOGS", "OTEL_TRACES_RETENTION_SECS", + "AI_SHARED_ARTIFACT_RETENTION_SECS", "DISABLE_S3_STORE", "PG_SCHEMA", "PG_LISTENER_REFRESH_PERIOD_SECS", diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index dce40048f5..8e17ec676e 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -151,6 +151,7 @@ pub const DEFAULT_HUB_BASE_URL: &str = "https://hub.windmill.dev"; pub const PRIVATE_HUB_MIN_VERSION: i32 = 10_000_000; pub const DEFAULT_SERVICE_LOG_RETENTION_SECS: i64 = 60 * 60 * 24 * 14; // 2 weeks retention period for logs pub const DEFAULT_OTEL_TRACES_RETENTION_SECS: i64 = 60 * 60 * 24 * 7; // 1 week retention period for HTTP request spans +pub const DEFAULT_AI_SHARED_ARTIFACT_RETENTION_SECS: i64 = 60 * 60 * 24 * 30; pub const WM_DEPLOYERS_GROUP: &str = "wm_deployers"; /// A century. Every consumer has to survive `now - retention`, and the ceilings are much lower @@ -229,6 +230,13 @@ pub fn service_log_retention_secs() -> i64 { SERVICE_LOG_RETENTION_SECS.load(std::sync::atomic::Ordering::Relaxed) } +/// How long a shared AI session artifact stays viewable, in seconds, counted from the last time +/// its author shared it. Read by both the API, which stops serving an expired share, and the +/// monitor, which deletes it — so both must agree, which is why they share this one reader. +pub fn ai_shared_artifact_retention_secs() -> i64 { + *AI_SHARED_ARTIFACT_RETENTION_SECS +} + /// Canonical form of a base URL, used as one of the inputs to the offline-license /// instance hash (`compute_instance_hash`). /// @@ -476,6 +484,15 @@ lazy_static::lazy_static! { /// [`set_otel_traces_retention_secs`] is the only writer, [`otel_traces_retention_secs`] the /// only reader. static ref OTEL_TRACES_RETENTION_SECS: AtomicI64 = AtomicI64::new(DEFAULT_OTEL_TRACES_RETENTION_SECS); + /// Read it with [`ai_shared_artifact_retention_secs`]. + static ref AI_SHARED_ARTIFACT_RETENTION_SECS: i64 = clamp_retention_secs( + std::env::var("AI_SHARED_ARTIFACT_RETENTION_SECS") + .ok() + .and_then(|s| s.parse::().ok()) + .unwrap_or(DEFAULT_AI_SHARED_ARTIFACT_RETENTION_SECS), + DEFAULT_AI_SHARED_ARTIFACT_RETENTION_SECS, + "AI shared artifact", + ); pub static ref MONITOR_LOGS_ON_OBJECT_STORE: AtomicBool = AtomicBool::new(false); diff --git a/frontend/src/lib/components/copilot/chat/LinkRenderer.svelte b/frontend/src/lib/components/copilot/chat/LinkRenderer.svelte index e93017b095..364e7fceaf 100644 --- a/frontend/src/lib/components/copilot/chat/LinkRenderer.svelte +++ b/frontend/src/lib/components/copilot/chat/LinkRenderer.svelte @@ -13,6 +13,7 @@ type WindmillItemKind, type WorkspaceItemTargetKind } from './workspaceItems.svelte' + import { safeHref } from './safeHref' type Props = { href?: string @@ -45,6 +46,8 @@ const previewAction = $derived(available?.type === 'open_item_preview' ? available : undefined) const drawerAction = $derived(available?.type === 'open_created_resource' ? available : undefined) + const allowedHref = $derived(safeHref(href, window.location.href)) + const modifier = newTabModifier() const hint = $derived( @@ -68,7 +71,7 @@ } -{#if href} +{#if allowedHref} {#if wmKind} {:else} - + {@render children?.()} {/if} +{:else} + + {@render children?.()} {/if} diff --git a/frontend/src/lib/components/copilot/chat/artifacts/ArtifactBody.svelte b/frontend/src/lib/components/copilot/chat/artifacts/ArtifactBody.svelte new file mode 100644 index 0000000000..7db5604775 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/artifacts/ArtifactBody.svelte @@ -0,0 +1,46 @@ + + + +{#if source} + + {#key content} + + {/key} +{:else} + +
    +
    + +
    +{/if} diff --git a/frontend/src/lib/components/copilot/chat/artifacts/ArtifactExportButton.svelte b/frontend/src/lib/components/copilot/chat/artifacts/ArtifactExportButton.svelte new file mode 100644 index 0000000000..9fc5f767b4 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/artifacts/ArtifactExportButton.svelte @@ -0,0 +1,42 @@ + + + + diff --git a/frontend/src/lib/components/copilot/chat/artifacts/ArtifactShareButton.svelte b/frontend/src/lib/components/copilot/chat/artifacts/ArtifactShareButton.svelte new file mode 100644 index 0000000000..4924473e98 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/artifacts/ArtifactShareButton.svelte @@ -0,0 +1,196 @@ + + + + {#snippet trigger()} + + {/snippet} + {#snippet content()} +
    +
    + Share with workspace + + {#if share} + Members of {workspace} can open a read-only copy of v{share.version} with this link. + {:else} + Members of {workspace} will be able to open a read-only copy of v{version} with a link. + {/if} + {#if status.current} + The copy is deleted {formatRetention(status.current.retention_secs)} after it is shared. + {/if} + +
    + + {#if share && url} +
    + + + Expires {displayDate(share.expires_at)} + +
    + {#if change} +
    + + {#if change === 'newer'} + The link shows v{share.version}; v{version} is on screen. + {:else if change === 'older'} + The link shows v{share.version}, newer than the v{version} on screen. + {:else} + The link still shows the old name, “{share.name}”. + {/if} + + +
    + {/if} +
    + +
    + {:else} +
    + +
    + {/if} +
    + {/snippet} +     diff --git a/frontend/src/lib/components/copilot/chat/artifacts/ArtifactViewer.svelte b/frontend/src/lib/components/copilot/chat/artifacts/ArtifactViewer.svelte index ebc46178d0..8331825ee2 100644 --- a/frontend/src/lib/components/copilot/chat/artifacts/ArtifactViewer.svelte +++ b/frontend/src/lib/components/copilot/chat/artifacts/ArtifactViewer.svelte @@ -1,23 +1,14 @@
    @@ -169,27 +144,22 @@ {/if}
    - - + /> + {#if canPreview} {#if restoringPin} - {:else if source} - - {#key `${artifact.id}:${pinnedContent ? `v${pinnedContent.version}` : artifact.updatedAt}`} - - {/key} {:else} - -
    -
    - -
    + {/if}
    diff --git a/frontend/src/lib/components/copilot/chat/artifacts/artifactSharing.test.ts b/frontend/src/lib/components/copilot/chat/artifacts/artifactSharing.test.ts new file mode 100644 index 0000000000..115343117a --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/artifacts/artifactSharing.test.ts @@ -0,0 +1,21 @@ +import { describe, expect, it, vi } from 'vitest' + +vi.mock('$lib/base', () => ({ base: '' })) + +import { shareWorkspaceId } from './artifactSharing' + +describe('shareWorkspaceId', () => { + it('shares a fork session into the topmost workspace the user still belongs to', () => { + const workspaces = [ + { id: 'prod' }, + { id: 'wm-fork-a', parent_workspace_id: 'prod' }, + { id: 'wm-fork-b', parent_workspace_id: 'wm-fork-a' } + ] + expect(shareWorkspaceId('wm-fork-b', workspaces)).toBe('prod') + }) + + it('stops below a parent the user is not a member of', () => { + const workspaces = [{ id: 'wm-fork-a', parent_workspace_id: 'prod' }] + expect(shareWorkspaceId('wm-fork-a', workspaces)).toBe('wm-fork-a') + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/artifacts/artifactSharing.ts b/frontend/src/lib/components/copilot/chat/artifacts/artifactSharing.ts new file mode 100644 index 0000000000..57b08cba9b --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/artifacts/artifactSharing.ts @@ -0,0 +1,42 @@ +import { base } from '$lib/base' + +/** + * The workspace a share from `workspaceId` lands in: the topmost ancestor the user still + * belongs to. A session often runs in a fork, whose members are its creator alone, so a link + * minted there would reach nobody — and it would be deleted with the fork. + */ +export function shareWorkspaceId( + workspaceId: string, + workspaces: { id: string; parent_workspace_id?: string | null }[] +): string { + let current = workspaceId + const seen = new Set([current]) + for (;;) { + const parent = workspaces.find((w) => w.id === current)?.parent_workspace_id + if (!parent || seen.has(parent) || !workspaces.some((w) => w.id === parent)) return current + seen.add(parent) + current = parent + } +} + +export function sharedArtifactUrl(workspaceId: string, shareId: string): string { + return `${window.location.origin}${base}/shared_artifacts/${encodeURIComponent( + shareId + )}?workspace=${encodeURIComponent(workspaceId)}` +} + +/** "30 days", "12 hours": the retention window in the largest whole unit it fills. */ +export function formatRetention(secs: number): string { + const units: [string, number][] = [ + ['day', 86400], + ['hour', 3600], + ['minute', 60] + ] + for (const [unit, size] of units) { + if (secs >= size) { + const n = Math.floor(secs / size) + return `${n} ${unit}${n === 1 ? '' : 's'}` + } + } + return `${secs} second${secs === 1 ? '' : 's'}` +} diff --git a/frontend/src/lib/components/copilot/chat/safeHref.test.ts b/frontend/src/lib/components/copilot/chat/safeHref.test.ts new file mode 100644 index 0000000000..5204ebe994 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/safeHref.test.ts @@ -0,0 +1,33 @@ +import { describe, expect, it } from 'vitest' +import { safeHref } from './safeHref' + +const BASE = 'https://app.example.com/sessions?workspace=demo' + +describe('safeHref', () => { + it.each([ + 'https://windmill.dev/docs', + 'http://localhost:3000/', + 'mailto:someone@example.com', + '/runs/abc', + '#anchor', + 'docs/page' + ])('keeps %s', (href) => { + expect(safeHref(href, BASE)).toBe(href) + }) + + it.each([ + 'javascript:alert(1)', + 'JavaScript:alert(1)', + ' javascript:alert(1)', + 'data:text/html,', + 'vbscript:msgbox', + 'file:///etc/passwd' + ])('drops %s', (href) => { + expect(safeHref(href, BASE)).toBeUndefined() + }) + + it('drops a missing or empty href', () => { + expect(safeHref(undefined, BASE)).toBeUndefined() + expect(safeHref('', BASE)).toBeUndefined() + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/safeHref.ts b/frontend/src/lib/components/copilot/chat/safeHref.ts new file mode 100644 index 0000000000..3acdabb464 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/safeHref.ts @@ -0,0 +1,17 @@ +const SAFE_PROTOCOLS = ['http:', 'https:', 'mailto:'] + +/** + * The href a rendered markdown link may carry, or undefined for one that must be dropped. + * + * Markdown reaches the chat renderers from a model, and from another member for a shared + * artifact, and `svelte-exmarkdown` passes `javascript:` and `data:` hrefs through untouched. + * Relative links resolve against `base` (the page), so they stay. + */ +export function safeHref(href: string | undefined, base: string): string | undefined { + if (!href) return undefined + try { + return SAFE_PROTOCOLS.includes(new URL(href, base).protocol) ? href : undefined + } catch { + return undefined + } +} diff --git a/frontend/src/routes/(root)/(logged)/shared_artifacts/[id]/+page.svelte b/frontend/src/routes/(root)/(logged)/shared_artifacts/[id]/+page.svelte new file mode 100644 index 0000000000..80449e6244 --- /dev/null +++ b/frontend/src/routes/(root)/(logged)/shared_artifacts/[id]/+page.svelte @@ -0,0 +1,144 @@ + + +
    +
    + {#if artifact} +
    +
    +
    + +

    + {artifact.name} +

    +
    + + Shared by {artifact.created_by} · v{artifact.version} · {displayDate( + artifact.shared_at + )} · expires {displayDate(artifact.expires_at)} + +
    +
    + {#if artifact.can_unshare} + + {/if} + + {#if artifact.kind === 'md'} + (showSource = v === 'source')} + > + {#snippet children({ item })} + + + {/snippet} + + {/if} +
    +
    +
    + +
    + {:else if shared.current?.state === 'gone'} + + {:else if shared.current?.state === 'error'} + + {:else} + + {/if} +
    +
    diff --git a/frontend/src/routes/(root)/(logged)/shared_artifacts/[id]/+page.ts b/frontend/src/routes/(root)/(logged)/shared_artifacts/[id]/+page.ts new file mode 100644 index 0000000000..efbac8862d --- /dev/null +++ b/frontend/src/routes/(root)/(logged)/shared_artifacts/[id]/+page.ts @@ -0,0 +1,5 @@ +export function load() { + return { + stuff: { title: 'Shared artifact' } + } +} From 5dcf40cb4f9d1d4d738c81cf57c3056e08eeadf0 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 14 Sep 2026 23:06:36 +0200 Subject: [PATCH 14/44] chore: move the EE pin forward to the commit that claims pending oauth accounts (#11127) Claude-Session: https://claude.ai/code/session_01GGciSSE5EFMiDf1dFWQq5M Co-authored-by: Claude Fable 5.1 --- backend/ee-repo-ref.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index a02a129b97..062211f925 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -04a9f1efb4a52c79fcd20258b34780c86103d27f +1ba6fe83451f0a1f8fafe04b7187087d51e0f769 From 75ee497011dca076de0923ded9da8e23e08bfb84 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 14 Sep 2026 23:16:37 +0200 Subject: [PATCH 15/44] fix(cli): stage a rewritten shared lockfile on git-sync deploy push (#11126) * fix(cli): stage a rewritten shared lockfile on git-sync deploy push Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01GGciSSE5EFMiDf1dFWQq5M * test(cli): pin that a swept shared lockfile is committed as a deletion Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01GGciSSE5EFMiDf1dFWQq5M * chore: bump the git sync hub script to 28969 Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01GGciSSE5EFMiDf1dFWQq5M --------- Co-authored-by: Claude Fable 5.1 --- backend/windmill-common/src/workspaces.rs | 2 +- cli/src/utils/git.ts | 6 ++ cli/test/gitsync_deploy_push_unit.test.ts | 98 +++++++++++++++++++++++ 3 files changed, 105 insertions(+), 1 deletion(-) create mode 100644 cli/test/gitsync_deploy_push_unit.test.ts diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index ad040c5905..3b00a87920 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -183,7 +183,7 @@ pub enum ObjectType { DatatableMigration, } -pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28958/sync-script-to-git-repo-windmill"; +pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28969/sync-script-to-git-repo-windmill"; /// Hub script that applies a repository's state back into a workspace /// (the repo → Windmill / "pull" direction). Same script the UI runs from diff --git a/cli/src/utils/git.ts b/cli/src/utils/git.ts index 5ca06f427a..354e15c36e 100644 --- a/cli/src/utils/git.ts +++ b/cli/src/utils/git.ts @@ -1,6 +1,7 @@ import * as log from "../core/log.ts"; import { execSync, spawnSync } from "node:child_process"; import { WM_FORK_PREFIX } from "../core/constants.ts"; +import { SHARED_LOCK_DIR } from "./script_common.ts"; // Fork *workspace id* prefix ("wm-fork-"). WM_FORK_PREFIX is the *branch* // prefix ("wm-fork") used inside the wm-fork// branch name. @@ -584,6 +585,11 @@ export function gitSyncDeployPush(params: { git(["add", "wmill-lock.yaml", `${parent_path}**`], { allowFail: true }); } } + // A shared lockfile (`dedupeLockfiles`) lives under `locks/`, outside every + // item's path glob, and the pull rewrites it when a deployed script's lock + // changed. `-A` also stages the deletion of a swept one; the add fails only + // when nothing under `locks/` exists or is tracked. + git(["add", "-A", "--", SHARED_LOCK_DIR], { allowFail: true }); // `git diff --cached --quiet` exits 1 iff there is something staged. const staged = git(["diff", "--cached", "--quiet"], { allowFail: true }); diff --git a/cli/test/gitsync_deploy_push_unit.test.ts b/cli/test/gitsync_deploy_push_unit.test.ts new file mode 100644 index 0000000000..f87e86bcc3 --- /dev/null +++ b/cli/test/gitsync_deploy_push_unit.test.ts @@ -0,0 +1,98 @@ +import { expect, test } from "bun:test"; +import { execFileSync } from "node:child_process"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { gitSyncDeployPush } from "../src/utils/git.ts"; + +function git(cwd: string, ...args: string[]): string { + return execFileSync("git", args, { cwd, encoding: "utf8" }).trim(); +} + +// A seeded clone of a bare remote, with `files` committed on main. +async function seededClone( + files: Record, +): Promise<{ bare: string; work: string }> { + const bare = await mkdtemp(join(tmpdir(), "wmill_deploy_push_bare_")); + execFileSync("git", ["init", "--quiet", "--bare", "--initial-branch=main", bare]); + const work = await mkdtemp(join(tmpdir(), "wmill_deploy_push_work_")); + git(work, "init", "--quiet", "--initial-branch=main"); + git(work, "config", "user.email", "seed@windmill.dev"); + git(work, "config", "user.name", "seed"); + for (const [path, content] of Object.entries(files)) { + await mkdir(join(work, path, ".."), { recursive: true }); + await writeFile(join(work, path), content); + } + git(work, "add", "-A"); + git(work, "commit", "--quiet", "-m", "seed"); + git(work, "remote", "add", "origin", `file://${bare}`); + git(work, "push", "--quiet", "-u", "origin", "main"); + return { bare, work }; +} + +function deployPushIn(work: string, path: string) { + const cwd = process.cwd(); + process.chdir(work); + try { + return gitSyncDeployPush({ + items: [{ path_type: "script", path, commit_msg: `deploy ${path}` }], + authorName: "windmill", + authorEmail: "windmill@windmill.dev", + }); + } finally { + process.chdir(cwd); + } +} + +test("a rewritten shared lockfile is committed with the deployed item", async () => { + const { bare, work } = await seededClone({ + "wmill-lock.yaml": "locks: {}\n", + "f/dd/a.script.yaml": "lock: '!inline locks/requirements.in.lock'\n", + "locks/requirements.in.lock": "requests==2.31.0\n", + }); + // What the deploy callback's pull leaves behind after `f/dd/a` was relocked: + // the item's own files are unchanged, only the shared file moved. + await writeFile(join(work, "locks/requirements.in.lock"), "requests==2.32.3\n"); + + expect(deployPushIn(work, "f/dd/a").pushed).toBe(true); + expect(git(work, "show", "--name-only", "--format=", "HEAD")).toBe( + "locks/requirements.in.lock", + ); + expect(git(bare, "cat-file", "-p", "main:locks/requirements.in.lock")).toBe( + "requests==2.32.3", + ); + + await rm(bare, { recursive: true, force: true }); + await rm(work, { recursive: true, force: true }); +}); + +test("a swept shared lockfile is committed as a deletion", async () => { + const { bare, work } = await seededClone({ + "wmill-lock.yaml": "locks: {}\n", + "f/dd/a.script.yaml": "lock: '!inline f/dd/a.script.lock'\n", + "f/dd/a.script.lock": "requests==2.31.0\n", + "locks/requirements.in.lock": "requests==2.31.0\n", + }); + // The pull removed the last shared lockfile, and `locks/` with it. + await rm(join(work, "locks"), { recursive: true, force: true }); + + expect(deployPushIn(work, "f/dd/a").pushed).toBe(true); + expect(git(bare, "ls-tree", "--name-only", "main", "locks/")).toBe(""); + + await rm(bare, { recursive: true, force: true }); + await rm(work, { recursive: true, force: true }); +}); + +test("a repository without shared lockfiles is left alone", async () => { + const { bare, work } = await seededClone({ + "wmill-lock.yaml": "locks: {}\n", + "f/dd/a.script.yaml": "lock: '!inline f/dd/a.script.lock'\n", + "f/dd/a.script.lock": "requests==2.31.0\n", + }); + + expect(deployPushIn(work, "f/dd/a").pushed).toBe(false); + expect(git(bare, "rev-parse", "main")).toBe(git(work, "rev-parse", "HEAD")); + + await rm(bare, { recursive: true, force: true }); + await rm(work, { recursive: true, force: true }); +}); From e3e638f7f587f0ee090d06436575b65a0860a855 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 15 Sep 2026 09:45:10 +0200 Subject: [PATCH 16/44] fix: skip instance group members that are not email addresses (#11128) * fix: skip instance group members that are not email addresses * fix: keep provisioned members whose address only proper_email accepts * fix: judge instance group members by a mirror of the usr email constraint * fix: fold ascii only in the proper_email mirror, like the constraint * fix: let the database judge which instance group members usr will store * fix: cut a derived username to the column width so a long local part can be provisioned * chore: move the ee pin to the scim member doc fix * chore: update ee-repo-ref to 0780955effb657807d14f0eb503cba1d49cee007 This commit updates the EE repository reference after PR #801 was merged in windmill-ee-private. Previous ee-repo-ref: ee6452d489563204a98df883703f78d5e74cdd69 New ee-repo-ref: 0780955effb657807d14f0eb503cba1d49cee007 Automated by sync-ee-ref workflow. --------- Co-authored-by: windmill-internal-app[bot] --- backend/Cargo.lock | 1 + backend/ee-repo-ref.txt | 2 +- backend/windmill-api-groups/Cargo.toml | 1 + backend/windmill-api-groups/src/groups.rs | 24 ++- .../tests/groups.rs | 137 ++++++++++++++++++ backend/windmill-api-users/src/users.rs | 10 +- backend/windmill-common/src/usernames.rs | 37 ++++- backend/windmill-common/src/users.rs | 29 ++++ .../tests/usr_accepts_email.rs | 59 ++++++++ 9 files changed, 291 insertions(+), 9 deletions(-) create mode 100644 backend/windmill-common/tests/usr_accepts_email.rs diff --git a/backend/Cargo.lock b/backend/Cargo.lock index a1a358c4e4..873d3d0590 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -15253,6 +15253,7 @@ dependencies = [ "serde_json", "sql-builder", "sqlx", + "tracing", "uuid", "windmill-api-auth", "windmill-api-workspaces", diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 062211f925..00bf0cbe8f 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -1ba6fe83451f0a1f8fafe04b7187087d51e0f769 +0780955effb657807d14f0eb503cba1d49cee007 diff --git a/backend/windmill-api-groups/Cargo.toml b/backend/windmill-api-groups/Cargo.toml index 856eaacc0a..35443008f0 100644 --- a/backend/windmill-api-groups/Cargo.toml +++ b/backend/windmill-api-groups/Cargo.toml @@ -29,4 +29,5 @@ serde.workspace = true serde_json.workspace = true sql-builder.workspace = true sqlx.workspace = true +tracing.workspace = true uuid.workspace = true diff --git a/backend/windmill-api-groups/src/groups.rs b/backend/windmill-api-groups/src/groups.rs index a15b9823d3..c931cf2255 100644 --- a/backend/windmill-api-groups/src/groups.rs +++ b/backend/windmill-api-groups/src/groups.rs @@ -22,7 +22,10 @@ use windmill_common::{ error::{Error, JsonResult, Result}, utils::{not_found_if_none, paginate, Pagination}, }; -use windmill_common::{db::UserDB, users::username_to_permissioned_as}; +use windmill_common::{ + db::UserDB, + users::{username_to_permissioned_as, usr_accepts_email}, +}; use serde::{Deserialize, Serialize}; use sqlx::{query_scalar, FromRow, Postgres, Transaction}; @@ -972,6 +975,15 @@ async fn add_user_igroup( ) -> Result { require_super_admin(&db, &authed).await?; + // `email_to_igroup` has no shape constraint of its own; `usr`, which the member is + // promoted into on reconcile, has `proper_email`, and a value failing it there would + // roll back every member of the group. + if !usr_accepts_email(&db, &email).await? { + return Err(Error::BadRequest(format!( + "'{email}' is not a valid email address" + ))); + } + let mut tx: Transaction<'_, Postgres> = db.begin().await?; // FOR UPDATE: the group row is the group-level mutex, taken before the workspace @@ -1424,6 +1436,16 @@ async fn overwrite_igroups( if let Some(emails) = &igroup.emails { for email in emails.iter() { + // An export can carry a member the source instance stored before ingest + // validated member values; it is dropped rather than failing the import. + if !usr_accepts_email(&mut *tx, email).await? { + tracing::warn!( + "Skipping member '{}' of imported instance group '{}': not an email address", + email, + igroup.name + ); + continue; + } sqlx::query!( "INSERT INTO email_to_igroup (email, igroup) VALUES ($1, $2)", email, diff --git a/backend/windmill-api-integration-tests/tests/groups.rs b/backend/windmill-api-integration-tests/tests/groups.rs index f86522aae3..26e5caeea1 100644 --- a/backend/windmill-api-integration-tests/tests/groups.rs +++ b/backend/windmill-api-integration-tests/tests/groups.rs @@ -913,3 +913,140 @@ async fn test_preserve_orphaned_members_migration(db: Pool) -> anyhow: Ok(()) } + +/// A membership row whose value is not an email (an IdP object id a SCIM sync stored before +/// member values were validated) must not break the workspace's instance-group save: the +/// reconciler skips it and still provisions the valid members. The admin endpoint refuses to +/// add such a value in the first place. +#[cfg(feature = "private")] +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_instance_group_member_that_is_not_an_email_is_skipped( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let global_base = format!("http://localhost:{port}/api/groups"); + let ws_base = format!("http://localhost:{port}/api/w/test-workspace/workspaces"); + const ENTRA_OBJECT_ID: &str = "ef40ea04-1a9e-4a84-9e65-cb1baa81dfed"; + + let resp = authed(client().post(format!("{global_base}/create"))) + .json(&json!({ "name": "entra_grp" })) + .send() + .await?; + assert_eq!(resp.status(), 200, "create"); + let resp = authed(client().post(format!("{global_base}/adduser/entra_grp"))) + .json(&json!({ "email": "kept@example.com" })) + .send() + .await?; + assert_eq!(resp.status(), 200, "adduser"); + + let resp = authed(client().post(format!("{global_base}/adduser/entra_grp"))) + .json(&json!({ "email": ENTRA_OBJECT_ID })) + .send() + .await?; + assert_eq!( + resp.status(), + 400, + "adduser must refuse a value that is not an email" + ); + let too_wide = format!("{}@example.com", "a".repeat(244)); + let resp = authed(client().post(format!("{global_base}/adduser/entra_grp"))) + .json(&json!({ "email": too_wide })) + .send() + .await?; + assert_eq!( + resp.status(), + 400, + "adduser must refuse a value wider than the email columns" + ); + // A valid address whose local part is wider than the username columns: the derived + // username is cut to fit rather than failing the promotion. + let long_local_part = format!("{}@example.com", "a".repeat(60)); + let resp = authed(client().post(format!("{global_base}/adduser/entra_grp"))) + .json(&json!({ "email": long_local_part })) + .send() + .await?; + assert_eq!(resp.status(), 200, "adduser long local part"); + + sqlx::query("INSERT INTO email_to_igroup (email, igroup) VALUES ($1, 'entra_grp')") + .bind(ENTRA_OBJECT_ID) + .execute(&db) + .await?; + + // A member whose address only the wider `proper_email` of `usr` accepts, already + // provisioned through the group: reconciliation must keep and re-role them, since + // removal destroys their drafts, inputs and permissions. + sqlx::raw_sql( + r#" + INSERT INTO email_to_igroup (email, igroup) VALUES ('"quoted"@example.com', 'entra_grp'); + INSERT INTO usr (workspace_id, username, email, is_admin, operator, added_via) + VALUES ('test-workspace', 'quoted', '"quoted"@example.com', false, true, + '{"source": "instance_group", "group": "entra_grp"}'::jsonb); + "#, + ) + .execute(&db) + .await?; + + let resp = authed(client().post(format!("{ws_base}/edit_instance_groups"))) + .json(&json!({ + "groups": ["entra_grp"], + "roles": { "entra_grp": "developer" } + })) + .send() + .await?; + assert_eq!(resp.status(), 200, "edit: {}", resp.text().await?); + + let mut members: Vec<(String, bool)> = sqlx::query_as( + "SELECT email, operator FROM usr WHERE workspace_id = 'test-workspace' + AND added_via->>'source' = 'instance_group'", + ) + .fetch_all(&db) + .await?; + members.sort(); + assert_eq!( + members, + vec![ + ("\"quoted\"@example.com".to_string(), false), + (long_local_part.clone(), false), + ("kept@example.com".to_string(), false), + ], + "valid members provisioned and existing member kept, all as developers; non-email one skipped" + ); + + // A full import carrying the same rows: the object id is dropped, the address only + // `proper_email` accepts is kept, and neither member loses their workspace row. + let resp = authed(client().post(format!("{global_base}/overwrite"))) + .json(&json!([{ + "name": "entra_grp", + "emails": ["kept@example.com", "\"quoted\"@example.com", long_local_part, ENTRA_OBJECT_ID] + }])) + .send() + .await?; + assert_eq!(resp.status(), 200, "overwrite: {}", resp.text().await?); + + let mut stored: Vec = + sqlx::query_scalar("SELECT email FROM email_to_igroup WHERE igroup = 'entra_grp'") + .fetch_all(&db) + .await?; + stored.sort(); + assert_eq!( + stored, + vec![ + "\"quoted\"@example.com".to_string(), + long_local_part.clone(), + "kept@example.com".to_string(), + ], + "import drops the object id and keeps the rest" + ); + let mut after_import: Vec<(String, bool)> = sqlx::query_as( + "SELECT email, operator FROM usr WHERE workspace_id = 'test-workspace' + AND added_via->>'source' = 'instance_group'", + ) + .fetch_all(&db) + .await?; + after_import.sort(); + assert_eq!(after_import, members, "import must not evict either member"); + + Ok(()) +} diff --git a/backend/windmill-api-users/src/users.rs b/backend/windmill-api-users/src/users.rs index f152ba8808..476a900ef6 100644 --- a/backend/windmill-api-users/src/users.rs +++ b/backend/windmill-api-users/src/users.rs @@ -53,8 +53,8 @@ use windmill_common::per_minute_counter::PerMinuteCounter; use windmill_common::users::truncate_token; use windmill_common::users::COOKIE_NAME; use windmill_common::users::{ - username_to_permissioned_as, PERMISSIONED_AS_MAX_LEN, SUPERADMIN_NOTIFICATION_EMAIL, - SUPERADMIN_SECRET_EMAIL, SUPERADMIN_SYNC_EMAIL, VALID_EMAIL, + username_to_permissioned_as, EMAIL_COLUMN_MAX_LEN, PERMISSIONED_AS_MAX_LEN, + SUPERADMIN_NOTIFICATION_EMAIL, SUPERADMIN_SECRET_EMAIL, SUPERADMIN_SYNC_EMAIL, VALID_EMAIL, }; use windmill_common::utils::paginate; use windmill_common::worker::CLOUD_HOSTED; @@ -1758,7 +1758,6 @@ struct ChangeUserEmail { /// `varchar(50)`, and `v2_job.permissioned_as` in a `varchar(55)`; every other email column is /// `varchar(255)`. The strictest of the two bounds is used for all of them. const SHORT_EMAIL_COLUMN_MAX_LEN: usize = 50; -const EMAIL_COLUMN_MAX_LEN: usize = 255; /// Move an account to a new email address, in place: the `password` row (and with it the /// instance-wide username, the role and the login type) is kept and every email-keyed row is @@ -3253,7 +3252,10 @@ mod same_origin_rd_tests { /// Both provisioning writes reference `password(email)`; a typo'd address from the /// provisioning script should read as "no such account", not as a foreign-key error. -async fn require_account(tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, email: &str) -> Result<()> { +async fn require_account( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + email: &str, +) -> Result<()> { let exists = sqlx::query_scalar!( "SELECT EXISTS(SELECT 1 FROM password WHERE email = $1)", email diff --git a/backend/windmill-common/src/usernames.rs b/backend/windmill-common/src/usernames.rs index 6fde2a8186..adb81ae682 100644 --- a/backend/windmill-common/src/usernames.rs +++ b/backend/windmill-common/src/usernames.rs @@ -17,6 +17,23 @@ lazy_static::lazy_static! { pub static ref VALID_USERNAME: Regex = Regex::new(r#"^[a-zA-Z][a-zA-Z_0-9]*$"#).unwrap(); } +/// Width of the `username` columns of `usr`, `password` and `pending_user`. +pub const USERNAME_MAX_LEN: usize = 50; + +/// `base` with the collision suffix of `attempt` appended (none for the first attempt), cut +/// to `USERNAME_MAX_LEN`. A local part longer than the column is a valid email, and an +/// insert that fails on the derived username rolls back everything around it. +pub fn fit_username(base: &str, attempt: u32) -> String { + let suffix = if attempt > 1 { + attempt.to_string() + } else { + String::new() + }; + let mut username: String = base.chars().take(USERNAME_MAX_LEN - suffix.len()).collect(); + username.push_str(&suffix); + username +} + pub async fn generate_instance_wide_unique_username<'c>( tx: &mut Transaction<'c, Postgres>, email: &str, @@ -41,9 +58,7 @@ pub async fn generate_instance_wide_unique_username<'c>( email ))); } - if i > 1 { - username = format!("{}{}", base_username, i) - } + username = fit_username(&base_username, i); username_conflict = sqlx::query_scalar!( "SELECT EXISTS(SELECT 1 FROM usr WHERE username = $1 and email != $2 UNION SELECT 1 FROM password WHERE username = $1 UNION SELECT 1 FROM pending_user WHERE username = $1)", &username, @@ -164,3 +179,19 @@ pub async fn get_instance_username_or_create_pending<'c>( } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_fit_username_keeps_the_column_width() { + assert_eq!(fit_username("alice", 1), "alice"); + assert_eq!(fit_username("alice", 2), "alice2"); + let base = "a".repeat(60); + assert_eq!(fit_username(&base, 1), "a".repeat(USERNAME_MAX_LEN)); + let with_suffix = fit_username(&base, 1000); + assert_eq!(with_suffix.len(), USERNAME_MAX_LEN); + assert!(with_suffix.ends_with("1000")); + } +} diff --git a/backend/windmill-common/src/users.rs b/backend/windmill-common/src/users.rs index 941328a8b2..1b696746b0 100644 --- a/backend/windmill-common/src/users.rs +++ b/backend/windmill-common/src/users.rs @@ -13,6 +13,35 @@ lazy_static::lazy_static! { pub static ref VALID_EMAIL: regex::Regex = regex::Regex::new( r"^[A-Za-z0-9!#$%&'*+/=?^_`{|}~-]+(\.[A-Za-z0-9!#$%&'*+/=?^_`{|}~-]+)*@([A-Za-z0-9]([A-Za-z0-9-]*[A-Za-z0-9])?\.)+[A-Za-z0-9]([A-Za-z0-9-]*[A-Za-z0-9])?$" ).unwrap(); + +} + +/// Width of the `email` columns of `usr`, `workspace_invite` and `email_to_igroup`. +pub const EMAIL_COLUMN_MAX_LEN: usize = 255; + +/// The regex of the `proper_email` CHECK constraint on `usr` and `workspace_invite` +/// (`20220620210708_regex_fix`), verbatim, for [`usr_accepts_email`]. Evaluated by the +/// database and never by a Rust engine: `~*` folds case under the database collation, so a +/// fixed mirror accepts addresses the constraint rejects, or rejects ones it holds, on some +/// locale. `windmill-common/tests/usr_accepts_email.rs` pins the text to the constraint. +pub const PROPER_EMAIL_PATTERN: &str = r#"^(?:[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\[(?:(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9]))\.){3}(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9])|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\])$"#; + +/// Whether `usr` (and `workspace_invite`) will store `email`: the `proper_email` regex as the +/// database evaluates it, plus the column width. Unlike [`VALID_EMAIL`] this admits every +/// address those tables already hold, which matters wherever an existing member is judged. +pub async fn usr_accepts_email<'c, E>(db: E, email: &str) -> crate::error::Result +where + E: sqlx::Executor<'c, Database = sqlx::Postgres>, +{ + if email.contains('\0') || email.chars().count() > EMAIL_COLUMN_MAX_LEN { + return Ok(false); + } + let accepted: bool = sqlx::query_scalar("SELECT $1::text ~* $2::text") + .bind(email) + .bind(PROPER_EMAIL_PATTERN) + .fetch_one(db) + .await?; + Ok(accepted) } pub const SUPERADMIN_SECRET_EMAIL: &str = "superadmin_secret@windmill.dev"; diff --git a/backend/windmill-common/tests/usr_accepts_email.rs b/backend/windmill-common/tests/usr_accepts_email.rs new file mode 100644 index 0000000000..624d8812b4 --- /dev/null +++ b/backend/windmill-common/tests/usr_accepts_email.rs @@ -0,0 +1,59 @@ +//! `usr_accepts_email` predicts whether `usr` will store an address by evaluating +//! `PROPER_EMAIL_PATTERN` in the database. It only stays right while that text matches the +//! `proper_email` constraint and the width matches the column: each sample below must be +//! stored by `usr` exactly when the check accepts it, and everything `VALID_EMAIL` accepts +//! within the width must be stored too. + +use sqlx::{Pool, Postgres}; +use windmill_common::users::{usr_accepts_email, EMAIL_COLUMN_MAX_LEN, VALID_EMAIL}; + +#[sqlx::test(migrations = "../migrations")] +async fn usr_accepts_email_agrees_with_the_constraint(db: Pool) -> anyhow::Result<()> { + let domain = "@example.com"; + let widest = format!( + "{}{domain}", + "a".repeat(EMAIL_COLUMN_MAX_LEN - domain.len()) + ); + let too_wide = format!("a{widest}"); + for email in [ + "alice@example.com", + "Alice@Example.COM", + "alice.bob+tag@sub.example.co.uk", + "\"quoted\"@example.com", + "\"quoted local\"@example.com", + "alice@[192.168.0.1]", + widest.as_str(), + too_wide.as_str(), + "ef40ea04-1a9e-4a84-9e65-cb1baa81dfed", + // Unicode case folding would map the long s and the Kelvin sign into `[a-z]`. + "u\u{17f}er@example.com", + "alice@example\u{212a}.com", + "alice", + "alice@example", + "alice@@example.com", + "alice @example.com", + "alice@example.com\nbob@example.com", + "", + ] { + let mut tx = db.begin().await?; + let stored = sqlx::query( + "INSERT INTO usr (workspace_id, username, email, is_admin, operator) + VALUES ('admins', 'probe', $1, false, false)", + ) + .bind(email) + .execute(&mut *tx) + .await + .is_ok(); + tx.rollback().await?; + + assert_eq!( + stored, + usr_accepts_email(&db, email).await?, + "{email:?}: `usr` and usr_accepts_email disagree" + ); + if VALID_EMAIL.is_match(email) && email.len() <= EMAIL_COLUMN_MAX_LEN { + assert!(stored, "{email:?}: VALID_EMAIL accepts what `usr` rejects"); + } + } + Ok(()) +} From 96963080f1711192fe1d9bc4142c1710511504d4 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 15 Sep 2026 09:45:50 +0200 Subject: [PATCH 17/44] fix(python): parse wheel RECORD paths as RFC 4180 csv fields (#11133) Co-authored-by: Claude Fable 5.1 --- .../windmill-worker/src/python_executor.rs | 73 ++++++++++++++++++- 1 file changed, 69 insertions(+), 4 deletions(-) diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 175ed8295d..5f5ca170b4 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -2338,6 +2338,33 @@ async fn spawn_uv_install( } } +/// First field (the path) of a wheel RECORD line. RECORD is CSV (PEP 376 / +/// RFC 4180): a path containing a comma or a double quote is written quoted, +/// with inner quotes doubled, so splitting on the first comma turns such an +/// entry into a name that never exists on disk. +fn record_first_field(line: &str) -> Option { + let Some(quoted) = line.strip_prefix('"') else { + return line + .split(',') + .next() + .filter(|p| !p.is_empty()) + .map(str::to_owned); + }; + let mut field = String::new(); + let mut chars = quoted.chars(); + while let Some(c) = chars.next() { + if c != '"' { + field.push(c); + } else if chars.as_str().starts_with('"') { + chars.next(); + field.push('"'); + } else { + return Some(field).filter(|f| !f.is_empty()); + } + } + None +} + /// Verify that every file listed in the wheel's RECORD exists on disk under /// `venv_p`. Used as a structural integrity check after both a successful /// `pull_from_tar` (object-store cache hit) and a successful local @@ -2386,9 +2413,9 @@ async fn verify_wheel_record(venv_p: &str) -> Result<(), String> { if trimmed.is_empty() { continue; } - let rel_path = match trimmed.split(',').next() { - Some(p) if !p.is_empty() => p, - _ => continue, + let rel_path = match record_first_field(trimmed) { + Some(p) => p, + None => continue, }; // Defensive: skip absolute paths or escaping entries — we only // validate package-relative files. @@ -2397,7 +2424,7 @@ async fn verify_wheel_record(venv_p: &str) -> Result<(), String> { } let full = format!("{venv_p}/{rel_path}"); if tokio::fs::metadata(&full).await.is_err() { - missing.push(rel_path.to_string()); + missing.push(rel_path); // Bound error size in pathological cases (e.g. wholly empty dir). if missing.len() >= 10 { missing.push("...".to_string()); @@ -3752,6 +3779,44 @@ mod tests { .is_ok()); } + #[tokio::test] + async fn test_verify_wheel_record_accepts_csv_quoted_path() { + let dir = tempfile::tempdir().unwrap(); + // A path containing a comma is CSV-quoted in RECORD (wcwidth 0.8.3 + // ships `wcwidth/textwrap.py,cover`). Splitting on the first comma + // looked for `"pkg/textwrap.py` and rejected a complete install. + write_fake_wheel( + dir.path(), + &["pkg/textwrap.py", "pkg/textwrap.py,cover"], + &[ + "pkg/textwrap.py,sha256=aaa,1", + "\"pkg/textwrap.py,cover\",sha256=bbb,1", + "pkg-1.0.0.dist-info/RECORD,,", + ], + ); + assert!(verify_wheel_record(dir.path().to_str().unwrap()) + .await + .is_ok()); + } + + #[test] + fn test_record_first_field_unquotes_rfc4180() { + assert_eq!( + record_first_field("pkg/a.py,sha256=x,1").as_deref(), + Some("pkg/a.py") + ); + assert_eq!( + record_first_field("\"pkg/a.py,cover\",sha256=x,1").as_deref(), + Some("pkg/a.py,cover") + ); + assert_eq!( + record_first_field("\"pkg/say \"\"hi\"\".py\",sha256=x,1").as_deref(), + Some("pkg/say \"hi\".py") + ); + assert_eq!(record_first_field(",,"), None); + assert_eq!(record_first_field("\"unterminated,sha256=x,1"), None); + } + // Regression tests for the concurrent-install guard. Two jobs installing the // same uncached dep into the shared `venv_p` used to race uv's `--reinstall`, // corrupting the on-disk wheel and failing with "Env installation did not From 082d8973282690d82b54af4670d5aec67c0de217 Mon Sep 17 00:00:00 2001 From: Alexander Petric Date: Tue, 15 Sep 2026 04:30:19 -0400 Subject: [PATCH 18/44] stop the variables page opening a drawer for the instance settings hash (#11129) Claude-Session: https://claude.ai/code/session_01LCt167dnWJ2EVnLpkHATh4 Co-authored-by: Claude Opus 5 (1M context) --- frontend/src/routes/(root)/(logged)/variables/+page.svelte | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frontend/src/routes/(root)/(logged)/variables/+page.svelte b/frontend/src/routes/(root)/(logged)/variables/+page.svelte index 52c248ea3e..873e812987 100644 --- a/frontend/src/routes/(root)/(logged)/variables/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/variables/+page.svelte @@ -264,7 +264,9 @@ let handledHash = '' $effect(() => { const hash = $page.url.hash - if (hash.length <= 1) { + // Only item paths are drawer targets: the same hash also carries global + // drawers like #superadmin-settings, which must not be looked up as a variable. + if (!/^#[ufg]\//.test(hash)) { // Navigating away from a drawer target must clear the tracker, or // re-targeting the same item later would be skipped as already handled. handledHash = '' From 69e6efd875e020779ea115c096331da8eae2ffe7 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Tue, 15 Sep 2026 10:34:57 +0200 Subject: [PATCH 19/44] fix(git-sync): run auto-pull as the admin who enabled it (#11121) * fix(git-sync): run auto-pull as the admin who enabled it Co-Authored-By: Claude Opus 5 * fix(git-sync): audit the admin grant fork pulls make Co-Authored-By: Claude Opus 5 * chore: bump ee ref for the post-commit fork grant audit Co-Authored-By: Claude Opus 5 * fix(git-sync): address review nits on the auto-pull stamp Co-Authored-By: Claude Opus 5 * chore: update ee-repo-ref to ccada062c072d7b74894b63863728fd1ef9bdffd This commit updates the EE repository reference after PR #799 was merged in windmill-ee-private. Previous ee-repo-ref: 7cee30f0cf12721cba551cd754dc817444810470 New ee-repo-ref: ccada062c072d7b74894b63863728fd1ef9bdffd Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Opus 5 Co-authored-by: windmill-internal-app[bot] --- ...e4e0b12b105d4475d7eba2d3a9573b93e388.json} | 4 +- ...246fd2bdf9e3fce89852120aa4ad4ad6abfc3.json | 26 --- ...0a7b681d101d286adfc6ff4548d1b9fdbab8c.json | 22 ++ ...dfec43dc8a042f9d95e63bf84b2dc15a72165.json | 23 +++ ...75f8d2f1665d329b8adea424b3a6f1e5c7013.json | 47 +++++ ...0baab26b8160929aea6d8f5f226e6ec4f8bd8.json | 24 +++ ...347e22196e46727487f35634af079c71c2bef.json | 23 +++ backend/Cargo.lock | 1 + backend/ee-repo-ref.txt | 2 +- .../fixtures/git_sync_autopull_identity.sql | 46 +++++ backend/tests/git_sync_autopull_identity.rs | 194 ++++++++++++++++++ .../tests/workspace_dependencies_git_sync.rs | 1 + .../windmill-api-workspaces/src/workspaces.rs | 39 +++- backend/windmill-api/openapi.yaml | 3 + backend/windmill-api/src/workspaces_export.rs | 9 +- backend/windmill-common/src/workspaces.rs | 7 + backend/windmill-git-sync/Cargo.toml | 5 +- docs/git-sync-pull-design.md | 29 +++ .../git_sync/GitSyncContext.svelte.ts | 11 +- .../git_sync/GitSyncRepositoryCard.svelte | 8 + 20 files changed, 481 insertions(+), 43 deletions(-) rename backend/.sqlx/{query-3202bed875693ae923f496272cd8ad89b2f17a9d3ef4659c2d2284415177b32c.json => query-0c4dc0e9dc159fac7e41492c78a4e4e0b12b105d4475d7eba2d3a9573b93e388.json} (54%) delete mode 100644 backend/.sqlx/query-17cdf02b4912078459526205849246fd2bdf9e3fce89852120aa4ad4ad6abfc3.json create mode 100644 backend/.sqlx/query-3fe41e2a72d02613a2b1c1c44fb0a7b681d101d286adfc6ff4548d1b9fdbab8c.json create mode 100644 backend/.sqlx/query-6dc8032100a28c4a6e843370038dfec43dc8a042f9d95e63bf84b2dc15a72165.json create mode 100644 backend/.sqlx/query-8bdfc02e7be54c2b610fed11cce75f8d2f1665d329b8adea424b3a6f1e5c7013.json create mode 100644 backend/.sqlx/query-a80a18774baf36d09b07da1e4e30baab26b8160929aea6d8f5f226e6ec4f8bd8.json create mode 100644 backend/.sqlx/query-b675c20bb7a15bec5e9a34d7ddf347e22196e46727487f35634af079c71c2bef.json create mode 100644 backend/tests/fixtures/git_sync_autopull_identity.sql create mode 100644 backend/tests/git_sync_autopull_identity.rs diff --git a/backend/.sqlx/query-3202bed875693ae923f496272cd8ad89b2f17a9d3ef4659c2d2284415177b32c.json b/backend/.sqlx/query-0c4dc0e9dc159fac7e41492c78a4e4e0b12b105d4475d7eba2d3a9573b93e388.json similarity index 54% rename from backend/.sqlx/query-3202bed875693ae923f496272cd8ad89b2f17a9d3ef4659c2d2284415177b32c.json rename to backend/.sqlx/query-0c4dc0e9dc159fac7e41492c78a4e4e0b12b105d4475d7eba2d3a9573b93e388.json index 0a833ba620..446c785770 100644 --- a/backend/.sqlx/query-3202bed875693ae923f496272cd8ad89b2f17a9d3ef4659c2d2284415177b32c.json +++ b/backend/.sqlx/query-0c4dc0e9dc159fac7e41492c78a4e4e0b12b105d4475d7eba2d3a9573b93e388.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT username, email FROM usr WHERE workspace_id = $1 AND is_admin = true AND operator = false AND disabled = false ORDER BY username LIMIT 1", + "query": "SELECT u.username, u.email FROM usr u WHERE u.workspace_id = $1 AND u.is_admin AND NOT u.operator AND NOT u.disabled AND NOT EXISTS (SELECT 1 FROM password p WHERE p.email = u.email AND p.disabled) ORDER BY u.username LIMIT 1", "describe": { "columns": [ { @@ -24,5 +24,5 @@ false ] }, - "hash": "3202bed875693ae923f496272cd8ad89b2f17a9d3ef4659c2d2284415177b32c" + "hash": "0c4dc0e9dc159fac7e41492c78a4e4e0b12b105d4475d7eba2d3a9573b93e388" } diff --git a/backend/.sqlx/query-17cdf02b4912078459526205849246fd2bdf9e3fce89852120aa4ad4ad6abfc3.json b/backend/.sqlx/query-17cdf02b4912078459526205849246fd2bdf9e3fce89852120aa4ad4ad6abfc3.json deleted file mode 100644 index bc2e6a6f63..0000000000 --- a/backend/.sqlx/query-17cdf02b4912078459526205849246fd2bdf9e3fce89852120aa4ad4ad6abfc3.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT COALESCE(username, split_part(email, '@', 1)) AS \"username!\", email FROM password WHERE super_admin = true AND disabled = false ORDER BY email LIMIT 1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "username!", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "email", - "type_info": "Varchar" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - null, - false - ] - }, - "hash": "17cdf02b4912078459526205849246fd2bdf9e3fce89852120aa4ad4ad6abfc3" -} diff --git a/backend/.sqlx/query-3fe41e2a72d02613a2b1c1c44fb0a7b681d101d286adfc6ff4548d1b9fdbab8c.json b/backend/.sqlx/query-3fe41e2a72d02613a2b1c1c44fb0a7b681d101d286adfc6ff4548d1b9fdbab8c.json new file mode 100644 index 0000000000..564c784d22 --- /dev/null +++ b/backend/.sqlx/query-3fe41e2a72d02613a2b1c1c44fb0a7b681d101d286adfc6ff4548d1b9fdbab8c.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT super_admin AS \"super_admin!\" FROM password WHERE email = $1 AND disabled = false", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "super_admin!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "3fe41e2a72d02613a2b1c1c44fb0a7b681d101d286adfc6ff4548d1b9fdbab8c" +} diff --git a/backend/.sqlx/query-6dc8032100a28c4a6e843370038dfec43dc8a042f9d95e63bf84b2dc15a72165.json b/backend/.sqlx/query-6dc8032100a28c4a6e843370038dfec43dc8a042f9d95e63bf84b2dc15a72165.json new file mode 100644 index 0000000000..8026b07416 --- /dev/null +++ b/backend/.sqlx/query-6dc8032100a28c4a6e843370038dfec43dc8a042f9d95e63bf84b2dc15a72165.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT r->'auto_pull'->>'enabled_by' AS \"enabled_by\"\n FROM workspace_settings, jsonb_array_elements(git_sync->'repositories') r\n WHERE workspace_id = $1 AND r->>'git_repo_resource_path' = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "enabled_by", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "6dc8032100a28c4a6e843370038dfec43dc8a042f9d95e63bf84b2dc15a72165" +} diff --git a/backend/.sqlx/query-8bdfc02e7be54c2b610fed11cce75f8d2f1665d329b8adea424b3a6f1e5c7013.json b/backend/.sqlx/query-8bdfc02e7be54c2b610fed11cce75f8d2f1665d329b8adea424b3a6f1e5c7013.json new file mode 100644 index 0000000000..0bcfd38acd --- /dev/null +++ b/backend/.sqlx/query-8bdfc02e7be54c2b610fed11cce75f8d2f1665d329b8adea424b3a6f1e5c7013.json @@ -0,0 +1,47 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT u.username, u.is_admin, u.operator, u.disabled, EXISTS (SELECT 1 FROM password p WHERE p.email = u.email AND p.disabled) AS \"instance_disabled!\" FROM usr u WHERE u.workspace_id = $1 AND u.email = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "username", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "is_admin", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "operator", + "type_info": "Bool" + }, + { + "ordinal": 3, + "name": "disabled", + "type_info": "Bool" + }, + { + "ordinal": 4, + "name": "instance_disabled!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false, + null + ] + }, + "hash": "8bdfc02e7be54c2b610fed11cce75f8d2f1665d329b8adea424b3a6f1e5c7013" +} diff --git a/backend/.sqlx/query-a80a18774baf36d09b07da1e4e30baab26b8160929aea6d8f5f226e6ec4f8bd8.json b/backend/.sqlx/query-a80a18774baf36d09b07da1e4e30baab26b8160929aea6d8f5f226e6ec4f8bd8.json new file mode 100644 index 0000000000..bc12fdba1f --- /dev/null +++ b/backend/.sqlx/query-a80a18774baf36d09b07da1e4e30baab26b8160929aea6d8f5f226e6ec4f8bd8.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO usr (workspace_id, username, email, is_admin, operator)\n SELECT $1::varchar, u.username, u.email, true, false FROM usr u\n WHERE u.workspace_id = $2 AND u.email = $3 AND u.is_admin AND NOT u.operator AND NOT u.disabled\n AND NOT EXISTS (SELECT 1 FROM password p WHERE p.email = u.email AND p.disabled)\n AND NOT EXISTS (SELECT 1 FROM usr f WHERE f.workspace_id = $1::varchar AND f.email = $3)\n ON CONFLICT DO NOTHING\n RETURNING username", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "username", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "a80a18774baf36d09b07da1e4e30baab26b8160929aea6d8f5f226e6ec4f8bd8" +} diff --git a/backend/.sqlx/query-b675c20bb7a15bec5e9a34d7ddf347e22196e46727487f35634af079c71c2bef.json b/backend/.sqlx/query-b675c20bb7a15bec5e9a34d7ddf347e22196e46727487f35634af079c71c2bef.json new file mode 100644 index 0000000000..f67a0f39f2 --- /dev/null +++ b/backend/.sqlx/query-b675c20bb7a15bec5e9a34d7ddf347e22196e46727487f35634af079c71c2bef.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS (SELECT 1 FROM usr WHERE workspace_id = $1 AND username = $2) AS \"claimed!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "claimed!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "b675c20bb7a15bec5e9a34d7ddf347e22196e46727487f35634af079c71c2bef" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 873d3d0590..3d1921ee76 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -15727,6 +15727,7 @@ dependencies = [ "tokio", "tracing", "uuid", + "windmill-audit", "windmill-common", "windmill-dep-map", "windmill-queue", diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 00bf0cbe8f..461156c173 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -0780955effb657807d14f0eb503cba1d49cee007 +ccada062c072d7b74894b63863728fd1ef9bdffd diff --git a/backend/tests/fixtures/git_sync_autopull_identity.sql b/backend/tests/fixtures/git_sync_autopull_identity.sql new file mode 100644 index 0000000000..3ae7d03d0e --- /dev/null +++ b/backend/tests/fixtures/git_sync_autopull_identity.sql @@ -0,0 +1,46 @@ +-- A parent workspace whose auto-pulled repository was last saved by alice, and a fork of +-- it holding only carol, the non-admin who created it. aaron is an admin who sorts before +-- alice; bob is no longer an admin; dora is a workspace admin deactivated on the instance. +-- sam and sue are instance superadmins who are not members: sam's instance username is +-- carol's, sue's is unclaimed. + +INSERT INTO workspace (id, name, owner) VALUES ('ap-parent', 'ap-parent', 'alice@windmill.dev'); +INSERT INTO workspace (id, name, owner, parent_workspace_id) + VALUES ('wm-fork-feat', 'feat', 'carol@windmill.dev', 'ap-parent'); + +INSERT INTO group_ (workspace_id, name, summary, extra_perms) VALUES + ('ap-parent', 'all', 'All users', '{}'), + ('wm-fork-feat', 'all', 'All users', '{}'); + +INSERT INTO password (email, password_hash, login_type, super_admin, verified, name, disabled) VALUES + ('aaron@windmill.dev', 'not-a-real-hash', 'password', false, true, 'aaron', false), + ('alice@windmill.dev', 'not-a-real-hash', 'password', false, true, 'alice', false), + ('bob@windmill.dev', 'not-a-real-hash', 'password', false, true, 'bob', false), + ('carol@windmill.dev', 'not-a-real-hash', 'password', false, true, 'carol', false), + ('dora@windmill.dev', 'not-a-real-hash', 'password', false, true, 'dora', true); + +INSERT INTO password (email, password_hash, login_type, super_admin, verified, name, disabled, username) VALUES + ('sam@windmill.dev', 'not-a-real-hash', 'password', true, true, 'sam', false, 'carol'), + ('sue@windmill.dev', 'not-a-real-hash', 'password', true, true, 'sue', false, 'sue'); + +INSERT INTO usr (workspace_id, email, username, is_admin) VALUES + ('ap-parent', 'aaron@windmill.dev', 'aaron', true), + ('ap-parent', 'alice@windmill.dev', 'alice', true), + ('ap-parent', 'bob@windmill.dev', 'bob', false), + ('ap-parent', 'carol@windmill.dev', 'carol', false), + ('ap-parent', 'dora@windmill.dev', 'dora', true), + ('wm-fork-feat', 'carol@windmill.dev', 'carol', false); + +INSERT INTO resource (workspace_id, path, value, resource_type, extra_perms, created_by) VALUES + ('ap-parent', 'u/alice/repo', '{"url": "https://github.com/test/repo.git", "branch": "main"}', + 'git_repository', '{}', 'alice'), + ('wm-fork-feat', 'u/alice/repo', '{"url": "https://github.com/test/repo.git", "branch": "main"}', + 'git_repository', '{}', 'alice'); + +INSERT INTO workspace_settings (workspace_id, git_sync) VALUES + ('ap-parent', '{"repositories":[{"git_repo_resource_path":"$res:u/alice/repo", + "use_individual_branch":false,"group_by_folder":false, + "auto_pull":{"enabled":true,"mode":"polling","sync_forks":true, + "enabled_by":"alice@windmill.dev"}}]}'), + ('wm-fork-feat', '{"repositories":[{"git_repo_resource_path":"$res:u/alice/repo", + "use_individual_branch":false,"group_by_folder":false}]}'); diff --git a/backend/tests/git_sync_autopull_identity.rs b/backend/tests/git_sync_autopull_identity.rs new file mode 100644 index 0000000000..4e7562c3f3 --- /dev/null +++ b/backend/tests/git_sync_autopull_identity.rs @@ -0,0 +1,194 @@ +//! An automatic pull runs as the admin stamped on the repository's settings, never as +//! someone picked from the workspace, and stops once that admin is revoked. A fork's +//! pull runs as the parent's pull identity, added to the fork first. +#![cfg(all(feature = "enterprise", feature = "private"))] + +use sqlx::{Pool, Postgres}; +use windmill_common::workspaces::GitRepositorySettings; +use windmill_git_sync::{reconcile_and_enqueue_pull, reconcile_fork_branch_pull}; + +const PARENT: &str = "ap-parent"; +const FORK: &str = "wm-fork-feat"; +const REPO: &str = "$res:u/alice/repo"; + +fn repo_enabled_by(email: &str) -> GitRepositorySettings { + serde_json::from_value(serde_json::json!({ + "git_repo_resource_path": REPO, + "use_individual_branch": false, + "group_by_folder": false, + "auto_pull": { "enabled": true, "enabled_by": email } + })) + .expect("repository settings") +} + +/// `(created_by, permissioned_as, permissioned_as_email)` of every pull job in `w_id`. +async fn pull_identities( + db: &Pool, + w_id: &str, +) -> anyhow::Result)>> { + Ok(sqlx::query_as( + "SELECT created_by, permissioned_as, permissioned_as_email FROM v2_job \ + WHERE workspace_id = $1 AND kind = 'deploymentcallback'", + ) + .bind(w_id) + .fetch_all(db) + .await?) +} + +fn identity(username: &str) -> (String, String, Option) { + ( + username.to_string(), + format!("u/{username}"), + Some(format!("{username}@windmill.dev")), + ) +} + +async fn recorded_pull_error(db: &Pool, w_id: &str) -> anyhow::Result { + let git_sync: serde_json::Value = + sqlx::query_scalar("SELECT git_sync FROM workspace_settings WHERE workspace_id = $1") + .bind(w_id) + .fetch_one(db) + .await?; + Ok( + git_sync["repositories"][0]["auto_pull"]["last_pull_status"]["error"] + .as_str() + .unwrap_or_default() + .to_string(), + ) +} + +#[sqlx::test(fixtures("git_sync_autopull_identity"))] +async fn pull_runs_as_the_admin_who_enabled_it(db: Pool) -> anyhow::Result<()> { + let job = reconcile_and_enqueue_pull( + &db, + PARENT, + &repo_enabled_by("alice@windmill.dev"), + "main", + "abc123", + None, + ) + .await?; + + assert!(job.is_some()); + assert_eq!(pull_identities(&db, PARENT).await?, vec![identity("alice")]); + Ok(()) +} + +/// bob was demoted in the workspace; dora is still a workspace admin but deactivated on +/// the instance. Neither may run the pull, and the failure lands on the status rather +/// than as an error, which a webhook delivery would turn into a failed response. +#[sqlx::test(fixtures("git_sync_autopull_identity"))] +async fn pull_fails_on_the_status_once_the_enabling_admin_is_revoked( + db: Pool, +) -> anyhow::Result<()> { + for email in ["bob@windmill.dev", "dora@windmill.dev"] { + let job = reconcile_and_enqueue_pull( + &db, + PARENT, + &repo_enabled_by(email), + "main", + "abc123", + None, + ) + .await?; + assert!(job.is_none(), "{email} must not run the pull"); + let error = recorded_pull_error(&db, PARENT).await?; + assert!(error.contains(email), "{error}"); + } + assert!(pull_identities(&db, PARENT).await?.is_empty()); + Ok(()) +} + +#[sqlx::test(fixtures("git_sync_autopull_identity"))] +async fn fork_pull_runs_as_the_parent_admin_added_to_the_fork( + db: Pool, +) -> anyhow::Result<()> { + let job = reconcile_fork_branch_pull(&db, PARENT, REPO, "wm-fork/main/feat", "main", "abc123") + .await?; + assert!( + job.is_some(), + "the fork branch must route to the fork and enqueue" + ); + + let (is_admin, in_all): (bool, bool) = sqlx::query_as( + "SELECT u.is_admin, EXISTS (SELECT 1 FROM usr_to_group g \ + WHERE g.workspace_id = u.workspace_id AND g.usr = u.username AND g.group_ = 'all') \ + FROM usr u WHERE u.workspace_id = $1 AND u.email = 'alice@windmill.dev'", + ) + .bind(FORK) + .fetch_one(&db) + .await?; + assert!( + is_admin && in_all, + "alice must be an admin member of the fork" + ); + assert_eq!(pull_identities(&db, FORK).await?, vec![identity("alice")]); + + let grants: i64 = sqlx::query_scalar( + "SELECT count(*) FROM audit_partitioned WHERE workspace_id = $1 \ + AND operation = 'users.git_sync_fork_add' AND resource = 'alice@windmill.dev'", + ) + .bind(FORK) + .fetch_one(&db) + .await?; + assert_eq!(grants, 1, "adding alice to the fork must be audited"); + Ok(()) +} + +/// A superadmin who is not a member runs the pull under their instance username, and +/// `u/` resolves through the workspace's members first. sam's instance username +/// is carol's, so sam's stamp must not run the pull as carol; sue's is unclaimed. +#[sqlx::test(fixtures("git_sync_autopull_identity"))] +async fn a_non_member_superadmin_runs_the_pull_only_under_an_unclaimed_username( + db: Pool, +) -> anyhow::Result<()> { + let job = reconcile_and_enqueue_pull( + &db, + PARENT, + &repo_enabled_by("sam@windmill.dev"), + "main", + "abc123", + None, + ) + .await?; + assert!(job.is_none(), "sam's username belongs to carol"); + assert!(pull_identities(&db, PARENT).await?.is_empty()); + + let job = reconcile_and_enqueue_pull( + &db, + PARENT, + &repo_enabled_by("sue@windmill.dev"), + "main", + "abc123", + None, + ) + .await?; + assert!(job.is_some()); + assert_eq!(pull_identities(&db, PARENT).await?, vec![identity("sue")]); + Ok(()) +} + +/// With no stamp on the parent, the fork pull still runs as the parent's first active +/// admin: the fork holds only its non-admin creator, so no identity resolved in the fork +/// could run it. +#[sqlx::test(fixtures("git_sync_autopull_identity"))] +async fn unstamped_fork_pull_runs_as_the_parents_first_admin( + db: Pool, +) -> anyhow::Result<()> { + sqlx::query( + "UPDATE workspace_settings SET git_sync = git_sync #- '{repositories,0,auto_pull,enabled_by}' \ + WHERE workspace_id = $1", + ) + .bind(PARENT) + .execute(&db) + .await?; + + let job = reconcile_fork_branch_pull(&db, PARENT, REPO, "wm-fork/main/feat", "main", "abc123") + .await?; + assert!( + job.is_some(), + "an unstamped parent must still sync its forks" + ); + assert_eq!(pull_identities(&db, FORK).await?, vec![identity("aaron")]); + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/workspace_dependencies_git_sync.rs b/backend/windmill-api-integration-tests/tests/workspace_dependencies_git_sync.rs index 4e25be1fa5..dc8e8dcc00 100644 --- a/backend/windmill-api-integration-tests/tests/workspace_dependencies_git_sync.rs +++ b/backend/windmill-api-integration-tests/tests/workspace_dependencies_git_sync.rs @@ -923,6 +923,7 @@ async fn test_pull_stays_on_the_workspace_lane(db: Pool) -> anyhow::Re &db, "test-workspace", &repo, + ("test-user", "test@windmill.dev"), None, false, None, diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 3a2b1d715a..640c0fe06a 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -910,12 +910,15 @@ fn redact_git_sync_webhook_secrets(git_sync: &mut serde_json::Value) { } /// Zero the server-owned auto-pull fields (webhook id/secret/url/error, synced -/// sha, last pull status) on a client-supplied `AutoPullSettings`. The client only -/// controls `enabled` / `mode` / `poll_interval_s`; the rest is written by the -/// server (webhook creation, poller) and must never be trusted from the request — -/// otherwise a caller could inject a webhook id/secret or fake sync state. -fn clear_client_supplied_auto_pull_state( +/// sha, last pull status) on a client-supplied `AutoPullSettings`, and stamp who +/// pulls run as: `saver_email` while auto pull is on. The client only controls +/// `enabled` / `mode` / `poll_interval_s`; the rest is written by the server (webhook +/// creation, poller, this save) and must never be trusted from the request — +/// otherwise a caller could inject a webhook id/secret, fake sync state, or pick who +/// pulls run as. +fn sanitize_client_auto_pull( auto_pull: &mut windmill_common::workspaces::AutoPullSettings, + saver_email: &str, ) { auto_pull.webhook_id = None; auto_pull.webhook_secret = None; @@ -923,6 +926,28 @@ fn clear_client_supplied_auto_pull_state( auto_pull.webhook_error = None; auto_pull.last_synced_sha = std::collections::HashMap::new(); auto_pull.last_pull_status = None; + auto_pull.enabled_by = auto_pull.enabled.then(|| saver_email.to_string()); +} + +#[cfg(test)] +mod sanitize_client_auto_pull_tests { + use windmill_common::workspaces::AutoPullSettings; + + #[test] + fn a_save_stamps_the_saver_over_any_client_supplied_stamp() { + let mut ap = AutoPullSettings { + enabled: true, + enabled_by: Some("forged@example.com".to_string()), + ..Default::default() + }; + super::sanitize_client_auto_pull(&mut ap, "saver@example.com"); + assert_eq!(ap.enabled_by.as_deref(), Some("saver@example.com")); + + ap.enabled = false; + ap.enabled_by = Some("forged@example.com".to_string()); + super::sanitize_client_auto_pull(&mut ap, "saver@example.com"); + assert_eq!(ap.enabled_by, None, "auto pull off carries no stamp"); + } } /// Whether a git-sync repository tracking `tracked` rules out `label_branch` as a dev workspace's @@ -3983,7 +4008,7 @@ async fn edit_git_sync_config( // stay clean. for repo in git_sync_settings.repositories.iter_mut() { if let Some(ap) = repo.auto_pull.as_mut() { - clear_client_supplied_auto_pull_state(ap); + sanitize_client_auto_pull(ap, &authed.email); } repo.open_pr_error = None; repo.credential = None; @@ -4230,7 +4255,7 @@ async fn edit_git_sync_repository( // existing repo re-derives it from the DB (carried over below) and a new one // starts clean. if let Some(ap) = new_config.repository.auto_pull.as_mut() { - clear_client_supplied_auto_pull_state(ap); + sanitize_client_auto_pull(ap, &authed.email); } new_config.repository.open_pr_error = None; new_config.repository.credential = None; diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 411d5f658c..b733157d85 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -35418,6 +35418,9 @@ components: type: string last_pull_status: $ref: "#/components/schemas/AutoPullStatus" + enabled_by: + type: string + description: Email of the admin automatic pulls apply changes as. Set by the server when the settings are saved. required: - enabled diff --git a/backend/windmill-api/src/workspaces_export.rs b/backend/windmill-api/src/workspaces_export.rs index 828509519e..a85f9d004e 100644 --- a/backend/windmill-api/src/workspaces_export.rs +++ b/backend/windmill-api/src/workspaces_export.rs @@ -1587,10 +1587,10 @@ pub(crate) async fn tarball_workspace( // Use v2 format only if explicitly requested, otherwise use v1 (legacy) for backward compatibility // Server-owned state (the HMAC webhook secret + hook id/error, the - // synced-sha / last-pull status, and what the credential check observed) - // must never leave the server: keep it out of export archives and synced - // repos, and don't let a re-imported workspace inherit another install's - // hook/sync state. Mirrors the GET-settings redaction. + // synced-sha / last-pull status, the admin automatic pulls run as, and what + // the credential check observed) must never leave the server: keep it out of + // export archives and synced repos, and don't let a re-imported workspace + // inherit another install's hook/sync state or pull identity. fn redact_git_sync_for_export(git_sync: Option) -> Option { let mut git_sync = git_sync?; if let Some(repos) = git_sync @@ -1607,6 +1607,7 @@ pub(crate) async fn tarball_workspace( "webhook_error", "last_synced_sha", "last_pull_status", + "enabled_by", ] { auto_pull.remove(field); } diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index 3b00a87920..9b6a471b2b 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -514,6 +514,11 @@ pub struct AutoPullSettings { pub last_synced_sha: std::collections::HashMap, #[serde(default, skip_serializing_if = "Option::is_none")] pub last_pull_status: Option, + /// Email of the admin this repository's automatic pulls (its own and its forks') + /// apply changes as: whoever last saved the settings with auto pull on. Stamped + /// server-side, never taken from the client. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub enabled_by: Option, } // Manual Debug so the HMAC `webhook_secret` (even encrypted) never lands in logs. @@ -524,6 +529,7 @@ impl std::fmt::Debug for AutoPullSettings { .field("mode", &self.mode) .field("poll_interval_s", &self.poll_interval_s) .field("sync_forks", &self.sync_forks) + .field("enabled_by", &self.enabled_by) .field("webhook_id", &self.webhook_id) .field( "webhook_secret", @@ -2709,6 +2715,7 @@ mod tests { webhook_secret: None, webhook_url: None, webhook_error: None, + enabled_by: None, last_synced_sha: synced .iter() .map(|(r, s)| (r.to_string(), s.to_string())) diff --git a/backend/windmill-git-sync/Cargo.toml b/backend/windmill-git-sync/Cargo.toml index f74581ba67..2b691ee48e 100644 --- a/backend/windmill-git-sync/Cargo.toml +++ b/backend/windmill-git-sync/Cargo.toml @@ -9,8 +9,8 @@ name = "windmill_git_sync" path = "./src/lib.rs" [features] -private = ["windmill-common/private", "windmill-dep-map/private"] -enterprise = ["windmill-queue/enterprise", "windmill-common/enterprise"] +private = ["windmill-common/private", "windmill-dep-map/private", "windmill-audit/private"] +enterprise = ["windmill-queue/enterprise", "windmill-common/enterprise", "windmill-audit/enterprise"] all_sqlx_features = ["enterprise"] default = [] @@ -23,5 +23,6 @@ tracing.workspace = true windmill-common = { workspace = true, default-features = false } windmill-queue.workspace = true windmill-dep-map.workspace = true +windmill-audit.workspace = true regex = "1.10.3" tokio = { workspace = true, features = ["full"] } \ No newline at end of file diff --git a/docs/git-sync-pull-design.md b/docs/git-sync-pull-design.md index 5f77a3f117..53d36c82fe 100644 --- a/docs/git-sync-pull-design.md +++ b/docs/git-sync-pull-design.md @@ -186,6 +186,35 @@ Routing — an event/poll result is `(repo, ref, head_sha, sender)`: filters): fan out, each workspace pulls with its own filters; `wmill.yaml` in the repo stays authoritative for include/exclude. +Identity — a pull applies changes as a real workspace admin, never a reserved identity. +The schedules, triggers and app policies it deploys persist their deployer as the identity +they run as, and `validate_on_behalf_of` refuses reserved sentinels there, so a real admin +is what keeps those deployable and revocable (demote or remove the admin and what runs +under them stops). + +- The admin is `auto_pull.enabled_by`, stamped server-side with the email of whoever last + saved the git sync settings with auto pull on. Re-saving as another admin rotates it. +- A stamp naming someone who is no longer an active admin (demoted, or deactivated in the + workspace or on the instance), and not an active instance superadmin either, fails the + pull rather than falling back to someone else. A superadmin who is not a member runs it + under their instance username, and only while no member of the workspace holds that + username: `u/` resolves through the workspace's members before the email. A + repository whose settings predate the stamp runs as the workspace's first active admin + until they are saved again. +- The identity is resolved before the deploy check is posted, and a failure to resolve it + or to enqueue is recorded on the repository's status, not returned: a returned error + would fail the webhook delivery, and hosts disable hooks whose deliveries keep failing. + The next push or poll retries. +- Fork pulls run as the parent repository's identity, stamped or not, resolved in the + parent (revoking that admin there stops fork pulls too), and first add that admin to + the fork as an admin member, since a plain fork carries only its creator. The fork's + owner cannot be the identity: a non-admin's `wmill sync push` diffs against what it can + see, so an item in a folder it cannot read reads as a create and the push fails on every + commit. CI tests do run as the owner (Phase 7), because they only execute. +- Known and accepted: repo writers control the pull's includes through `wmill.yaml`, so a + fork's owner can commit a user file that makes them admin of the fork and read the + parent secrets it cloned, as with the `push-on-merge-to-forks` Action this replaces. + Loop prevention (pull → deploys → deployment callback → commit → push event): 1. Skip events whose sender is the app bot (`windmill-sync-helper[bot]` / diff --git a/frontend/src/lib/components/git_sync/GitSyncContext.svelte.ts b/frontend/src/lib/components/git_sync/GitSyncContext.svelte.ts index 0150e329ce..075c6b1f24 100644 --- a/frontend/src/lib/components/git_sync/GitSyncContext.svelte.ts +++ b/frontend/src/lib/components/git_sync/GitSyncContext.svelte.ts @@ -1,5 +1,5 @@ import { getContext, setContext } from 'svelte' -import { enterpriseLicense } from '$lib/stores' +import { enterpriseLicense, userStore } from '$lib/stores' import { get } from 'svelte/store' import { sendUserToast } from '$lib/toast' import { apiErrorMessage } from '$lib/utils' @@ -534,6 +534,15 @@ export function createGitSyncContext(workspace: string) { } }) + // The server stamps the saving admin as who pulls run as; mirror it so the card + // names them without a reload. + if (repoToSave.auto_pull) { + repoToSave.auto_pull = { + ...repoToSave.auto_pull, + enabled_by: repoToSave.auto_pull.enabled ? get(userStore)?.email : undefined + } + } + // Update local state with migrated repository repositories[idx] = repoToSave initialRepositories[idx] = { ...repoToSave } diff --git a/frontend/src/lib/components/git_sync/GitSyncRepositoryCard.svelte b/frontend/src/lib/components/git_sync/GitSyncRepositoryCard.svelte index 4b4bff2168..32a510c54a 100644 --- a/frontend/src/lib/components/git_sync/GitSyncRepositoryCard.svelte +++ b/frontend/src/lib/components/git_sync/GitSyncRepositoryCard.svelte @@ -1038,6 +1038,14 @@ : 'Checking the tracked branch about every minute. New commits deploy automatically.'} {/if}
    + {#if repo.auto_pull?.enabled_by} +
    + Pulls apply changes as {repo.auto_pull.enabled_by}, the admin who last saved + these settings{repo.auto_pull.sync_forks + ? ', in this workspace and its forks' + : ''}. +
    + {/if} {#if hasManagedCredential && repo.auto_pull?.webhook_error}
    From 42f489685bc87a8479818175c87b5a21b0c17998 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Tue, 15 Sep 2026 11:32:50 +0200 Subject: [PATCH 20/44] feat: store resource type display names and label hub integrations (#11113) * feat: label resource types and integrations with hub display names Co-Authored-By: Claude Opus 5 (1M context) * fix: load hub integration names in the app and flow pickers Co-Authored-By: Claude Opus 5 (1M context) * fix: load hub resource type names where drawers title a type Co-Authored-By: Claude Opus 5 (1M context) * feat: store resource type display names and drop the hardcoded list Co-Authored-By: Claude Opus 5 (1M context) * fix: leave display_name out of the fork comparison Co-Authored-By: Claude Opus 5 (1M context) * fix: ignore over-long synced display names, move name loaders Co-Authored-By: Claude Opus 5 (1M context) * fix: share the hub integration list cache, backfill admins only Co-Authored-By: Claude Opus 5 (1M context) * fix: keep a name over a nameless duplicate, retry failed hub reads Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Opus 5 (1M context) --- ...04befc3a6596c258086805dd42f513ddb3ead.json | 20 ++++ ...6b07cdb70994be5724f08f864d29517f4907.json} | 12 +- ...c5dd6c7b0da4f76131313d3e6d97f36c5c34.json} | 6 +- ...ecdad014ce319066eabec0f0d7f53338698a.json} | 12 +- ...f4d070f803b6192739e9ebaad7427ce59251.json} | 12 +- ...e795ff9fb6b863dfe9c0bcad382c3b73a25b.json} | 12 +- ...76488dc6c80428e13e5db3c016e37072cfe4.json} | 4 +- ...2e2e8bc43633707d8365f73130c5bca3923b9.json | 18 --- ...95bd6b3fb6ebf3ba115160573487cee4a606.json} | 7 +- ...131148_resource_type_display_name.down.sql | 1 + ...14131148_resource_type_display_name.up.sql | 24 ++++ backend/src/main.rs | 67 +++++++++-- backend/summarized_schema.txt | 2 +- backend/windmill-api-embeddings/src/lib.rs | 2 +- .../tests/resources.rs | 24 ++++ backend/windmill-api-settings/src/lib.rs | 30 ++++- .../windmill-api-workspaces/src/workspaces.rs | 4 +- backend/windmill-api/openapi.yaml | 15 +++ backend/windmill-api/src/workspaces_export.rs | 2 +- backend/windmill-store/src/resources.rs | 42 ++++++- cli/src/commands/hub/hub.ts | 7 +- .../commands/resource-type/resource-type.ts | 1 + .../src/lib/components/AppConnectInner.svelte | 4 + .../src/lib/components/ImportSetupStep.svelte | 19 +++- .../LightweightResourcePicker.svelte | 2 + .../components/ResourceEditorDrawer.svelte | 3 + .../lib/components/ResourceTypePicker.svelte | 7 +- .../src/lib/components/displayNameLoaders.ts | 71 ++++++++++++ .../flows/pickers/PickHubApp.svelte | 2 + .../flows/pickers/PickHubFlow.svelte | 2 + .../flows/pickers/PickHubScript.svelte | 5 +- .../flows/pickers/PickHubScriptQuick.svelte | 12 +- .../lib/components/home/ListFilters.svelte | 3 +- .../components/home/ListFiltersQuick.svelte | 3 +- .../components/mcp/McpScopeSelector.svelte | 15 +-- .../src/lib/components/resourceTypeDisplay.ts | 105 +++++++++++++----- .../components/resourceTypeMatchRank.test.ts | 46 +++++++- .../(root)/(logged)/resources/+page.svelte | 15 +-- 38 files changed, 522 insertions(+), 116 deletions(-) create mode 100644 backend/.sqlx/query-212bf5b32de102a9c537907aaff04befc3a6596c258086805dd42f513ddb3ead.json rename backend/.sqlx/{query-623b061ccaa6bb883e95771fde8c911a165c9c430b7db389370361ca74d737f4.json => query-36ddecbdad3cce7a2593171ff10a6b07cdb70994be5724f08f864d29517f4907.json} (76%) rename backend/.sqlx/{query-8ad79b80033b38ebddf6c8cd4d8cb160d41bac4c45a0fc74d9c9e96d3ef4486a.json => query-386e14cf7572027f2c4ef313cd7cc5dd6c7b0da4f76131313d3e6d97f36c5c34.json} (70%) rename backend/.sqlx/{query-d0a95698b9a2c5e2543e94276d854d7e509c7db2c2ac7d395b7b53ad5dbc25e6.json => query-38b6c6cb91d3ba38838a7a015c59ecdad014ce319066eabec0f0d7f53338698a.json} (75%) rename backend/.sqlx/{query-e253b9e7e6450652589d6ee7ffa86d600e449cd399ac781af8b40c1c444972c3.json => query-6f993567336a2f5ff642ed54e3aaf4d070f803b6192739e9ebaad7427ce59251.json} (78%) rename backend/.sqlx/{query-45d5e9ead8193a04fd00c44a488590fdd2f7c4de45117a18360651655d153545.json => query-82350027cf9722a993f27808e570e795ff9fb6b863dfe9c0bcad382c3b73a25b.json} (78%) rename backend/.sqlx/{query-1c2157ce14e90f0751d7f0a9f2dbb3c5a5789a32423e75260098a5300a4af986.json => query-86ad1e7ebe659f97877cc142c09676488dc6c80428e13e5db3c016e37072cfe4.json} (52%) delete mode 100644 backend/.sqlx/query-972df41db505fbbd20a558b200a2e2e8bc43633707d8365f73130c5bca3923b9.json rename backend/.sqlx/{query-5899c7614f195fdd23e38389e52b004f957aafa2201b80638b5f87a625373f00.json => query-dd6f4b505f4c1e2c734c5d04528c95bd6b3fb6ebf3ba115160573487cee4a606.json} (64%) create mode 100644 backend/migrations/20260914131148_resource_type_display_name.down.sql create mode 100644 backend/migrations/20260914131148_resource_type_display_name.up.sql create mode 100644 frontend/src/lib/components/displayNameLoaders.ts diff --git a/backend/.sqlx/query-212bf5b32de102a9c537907aaff04befc3a6596c258086805dd42f513ddb3ead.json b/backend/.sqlx/query-212bf5b32de102a9c537907aaff04befc3a6596c258086805dd42f513ddb3ead.json new file mode 100644 index 0000000000..04b23832d9 --- /dev/null +++ b/backend/.sqlx/query-212bf5b32de102a9c537907aaff04befc3a6596c258086805dd42f513ddb3ead.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO resource_type (workspace_id, name, schema, description, format_extension, display_name, edited_at)\n VALUES ('admins', $1, $2, $3, $4, $6, now())\n ON CONFLICT (workspace_id, name) DO UPDATE\n SET schema = EXCLUDED.schema, description = EXCLUDED.description,\n -- A fileset is a set of files, so it cannot also be one file.\n -- Create and update reject the pair; this writer bypasses both, so\n -- it declines the extension rather than persisting the forbidden\n -- combination onto a same-named local fileset.\n format_extension = CASE\n WHEN resource_type.is_fileset THEN NULL\n WHEN $5 THEN EXCLUDED.format_extension\n ELSE resource_type.format_extension END,\n display_name = CASE WHEN $7 THEN EXCLUDED.display_name ELSE resource_type.display_name END,\n edited_at = now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Jsonb", + "Text", + "Varchar", + "Bool", + "Varchar", + "Bool" + ] + }, + "nullable": [] + }, + "hash": "212bf5b32de102a9c537907aaff04befc3a6596c258086805dd42f513ddb3ead" +} diff --git a/backend/.sqlx/query-623b061ccaa6bb883e95771fde8c911a165c9c430b7db389370361ca74d737f4.json b/backend/.sqlx/query-36ddecbdad3cce7a2593171ff10a6b07cdb70994be5724f08f864d29517f4907.json similarity index 76% rename from backend/.sqlx/query-623b061ccaa6bb883e95771fde8c911a165c9c430b7db389370361ca74d737f4.json rename to backend/.sqlx/query-36ddecbdad3cce7a2593171ff10a6b07cdb70994be5724f08f864d29517f4907.json index d2084d76d8..76e897d3a4 100644 --- a/backend/.sqlx/query-623b061ccaa6bb883e95771fde8c911a165c9c430b7db389370361ca74d737f4.json +++ b/backend/.sqlx/query-36ddecbdad3cce7a2593171ff10a6b07cdb70994be5724f08f864d29517f4907.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset from resource_type WHERE name = $1 AND (workspace_id = $2 OR workspace_id = 'admins')", + "query": "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset, display_name from resource_type WHERE name = $1 AND (workspace_id = $2 OR workspace_id = 'admins')", "describe": { "columns": [ { @@ -42,6 +42,11 @@ "ordinal": 7, "name": "is_fileset", "type_info": "Bool" + }, + { + "ordinal": 8, + "name": "display_name", + "type_info": "Varchar" } ], "parameters": { @@ -58,8 +63,9 @@ true, true, true, - false + false, + true ] }, - "hash": "623b061ccaa6bb883e95771fde8c911a165c9c430b7db389370361ca74d737f4" + "hash": "36ddecbdad3cce7a2593171ff10a6b07cdb70994be5724f08f864d29517f4907" } diff --git a/backend/.sqlx/query-8ad79b80033b38ebddf6c8cd4d8cb160d41bac4c45a0fc74d9c9e96d3ef4486a.json b/backend/.sqlx/query-386e14cf7572027f2c4ef313cd7cc5dd6c7b0da4f76131313d3e6d97f36c5c34.json similarity index 70% rename from backend/.sqlx/query-8ad79b80033b38ebddf6c8cd4d8cb160d41bac4c45a0fc74d9c9e96d3ef4486a.json rename to backend/.sqlx/query-386e14cf7572027f2c4ef313cd7cc5dd6c7b0da4f76131313d3e6d97f36c5c34.json index 3b9a2e2f34..2f794e6be0 100644 --- a/backend/.sqlx/query-8ad79b80033b38ebddf6c8cd4d8cb160d41bac4c45a0fc74d9c9e96d3ef4486a.json +++ b/backend/.sqlx/query-386e14cf7572027f2c4ef313cd7cc5dd6c7b0da4f76131313d3e6d97f36c5c34.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT EXISTS(SELECT 1 FROM resource_type WHERE workspace_id = 'admins' AND name = $1 AND schema IS NOT DISTINCT FROM $2 AND description IS NOT DISTINCT FROM $3 AND ($5 IS NOT TRUE OR format_extension IS NOT DISTINCT FROM $4))", + "query": "SELECT EXISTS(SELECT 1 FROM resource_type WHERE workspace_id = 'admins' AND name = $1 AND schema IS NOT DISTINCT FROM $2 AND description IS NOT DISTINCT FROM $3 AND ($5 IS NOT TRUE OR format_extension IS NOT DISTINCT FROM $4) AND ($7 IS NOT TRUE OR display_name IS NOT DISTINCT FROM $6))", "describe": { "columns": [ { @@ -15,6 +15,8 @@ "Jsonb", "Text", "Text", + "Bool", + "Text", "Bool" ] }, @@ -22,5 +24,5 @@ null ] }, - "hash": "8ad79b80033b38ebddf6c8cd4d8cb160d41bac4c45a0fc74d9c9e96d3ef4486a" + "hash": "386e14cf7572027f2c4ef313cd7cc5dd6c7b0da4f76131313d3e6d97f36c5c34" } diff --git a/backend/.sqlx/query-d0a95698b9a2c5e2543e94276d854d7e509c7db2c2ac7d395b7b53ad5dbc25e6.json b/backend/.sqlx/query-38b6c6cb91d3ba38838a7a015c59ecdad014ce319066eabec0f0d7f53338698a.json similarity index 75% rename from backend/.sqlx/query-d0a95698b9a2c5e2543e94276d854d7e509c7db2c2ac7d395b7b53ad5dbc25e6.json rename to backend/.sqlx/query-38b6c6cb91d3ba38838a7a015c59ecdad014ce319066eabec0f0d7f53338698a.json index ffd1670c04..87d740a247 100644 --- a/backend/.sqlx/query-d0a95698b9a2c5e2543e94276d854d7e509c7db2c2ac7d395b7b53ad5dbc25e6.json +++ b/backend/.sqlx/query-38b6c6cb91d3ba38838a7a015c59ecdad014ce319066eabec0f0d7f53338698a.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset from resource_type WHERE (workspace_id = $1 OR workspace_id = 'admins') ORDER BY name", + "query": "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset, display_name from resource_type WHERE (workspace_id = $1 OR workspace_id = 'admins') ORDER BY name", "describe": { "columns": [ { @@ -42,6 +42,11 @@ "ordinal": 7, "name": "is_fileset", "type_info": "Bool" + }, + { + "ordinal": 8, + "name": "display_name", + "type_info": "Varchar" } ], "parameters": { @@ -57,8 +62,9 @@ true, true, true, - false + false, + true ] }, - "hash": "d0a95698b9a2c5e2543e94276d854d7e509c7db2c2ac7d395b7b53ad5dbc25e6" + "hash": "38b6c6cb91d3ba38838a7a015c59ecdad014ce319066eabec0f0d7f53338698a" } diff --git a/backend/.sqlx/query-e253b9e7e6450652589d6ee7ffa86d600e449cd399ac781af8b40c1c444972c3.json b/backend/.sqlx/query-6f993567336a2f5ff642ed54e3aaf4d070f803b6192739e9ebaad7427ce59251.json similarity index 78% rename from backend/.sqlx/query-e253b9e7e6450652589d6ee7ffa86d600e449cd399ac781af8b40c1c444972c3.json rename to backend/.sqlx/query-6f993567336a2f5ff642ed54e3aaf4d070f803b6192739e9ebaad7427ce59251.json index c44d3d711d..e444c0d549 100644 --- a/backend/.sqlx/query-e253b9e7e6450652589d6ee7ffa86d600e449cd399ac781af8b40c1c444972c3.json +++ b/backend/.sqlx/query-6f993567336a2f5ff642ed54e3aaf4d070f803b6192739e9ebaad7427ce59251.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset from resource_type ORDER BY name", + "query": "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset, display_name from resource_type ORDER BY name", "describe": { "columns": [ { @@ -42,6 +42,11 @@ "ordinal": 7, "name": "is_fileset", "type_info": "Bool" + }, + { + "ordinal": 8, + "name": "display_name", + "type_info": "Varchar" } ], "parameters": { @@ -55,8 +60,9 @@ true, true, true, - false + false, + true ] }, - "hash": "e253b9e7e6450652589d6ee7ffa86d600e449cd399ac781af8b40c1c444972c3" + "hash": "6f993567336a2f5ff642ed54e3aaf4d070f803b6192739e9ebaad7427ce59251" } diff --git a/backend/.sqlx/query-45d5e9ead8193a04fd00c44a488590fdd2f7c4de45117a18360651655d153545.json b/backend/.sqlx/query-82350027cf9722a993f27808e570e795ff9fb6b863dfe9c0bcad382c3b73a25b.json similarity index 78% rename from backend/.sqlx/query-45d5e9ead8193a04fd00c44a488590fdd2f7c4de45117a18360651655d153545.json rename to backend/.sqlx/query-82350027cf9722a993f27808e570e795ff9fb6b863dfe9c0bcad382c3b73a25b.json index e4db87ec7d..9f7be9d7cb 100644 --- a/backend/.sqlx/query-45d5e9ead8193a04fd00c44a488590fdd2f7c4de45117a18360651655d153545.json +++ b/backend/.sqlx/query-82350027cf9722a993f27808e570e795ff9fb6b863dfe9c0bcad382c3b73a25b.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset FROM resource_type WHERE workspace_id = $1", + "query": "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset, display_name FROM resource_type WHERE workspace_id = $1", "describe": { "columns": [ { @@ -42,6 +42,11 @@ "ordinal": 7, "name": "is_fileset", "type_info": "Bool" + }, + { + "ordinal": 8, + "name": "display_name", + "type_info": "Varchar" } ], "parameters": { @@ -57,8 +62,9 @@ true, true, true, - false + false, + true ] }, - "hash": "45d5e9ead8193a04fd00c44a488590fdd2f7c4de45117a18360651655d153545" + "hash": "82350027cf9722a993f27808e570e795ff9fb6b863dfe9c0bcad382c3b73a25b" } diff --git a/backend/.sqlx/query-1c2157ce14e90f0751d7f0a9f2dbb3c5a5789a32423e75260098a5300a4af986.json b/backend/.sqlx/query-86ad1e7ebe659f97877cc142c09676488dc6c80428e13e5db3c016e37072cfe4.json similarity index 52% rename from backend/.sqlx/query-1c2157ce14e90f0751d7f0a9f2dbb3c5a5789a32423e75260098a5300a4af986.json rename to backend/.sqlx/query-86ad1e7ebe659f97877cc142c09676488dc6c80428e13e5db3c016e37072cfe4.json index fc354fb9a2..f417ab94f9 100644 --- a/backend/.sqlx/query-1c2157ce14e90f0751d7f0a9f2dbb3c5a5789a32423e75260098a5300a4af986.json +++ b/backend/.sqlx/query-86ad1e7ebe659f97877cc142c09676488dc6c80428e13e5db3c016e37072cfe4.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO resource_type (workspace_id, name, schema, description, edited_at, created_by, format_extension, is_fileset)\n SELECT $2, name, schema, description, edited_at, created_by, format_extension, is_fileset\n FROM resource_type\n WHERE workspace_id = $1", + "query": "INSERT INTO resource_type (workspace_id, name, schema, description, edited_at, created_by, format_extension, is_fileset, display_name)\n SELECT $2, name, schema, description, edited_at, created_by, format_extension, is_fileset, display_name\n FROM resource_type\n WHERE workspace_id = $1", "describe": { "columns": [], "parameters": { @@ -11,5 +11,5 @@ }, "nullable": [] }, - "hash": "1c2157ce14e90f0751d7f0a9f2dbb3c5a5789a32423e75260098a5300a4af986" + "hash": "86ad1e7ebe659f97877cc142c09676488dc6c80428e13e5db3c016e37072cfe4" } diff --git a/backend/.sqlx/query-972df41db505fbbd20a558b200a2e2e8bc43633707d8365f73130c5bca3923b9.json b/backend/.sqlx/query-972df41db505fbbd20a558b200a2e2e8bc43633707d8365f73130c5bca3923b9.json deleted file mode 100644 index 3624a32893..0000000000 --- a/backend/.sqlx/query-972df41db505fbbd20a558b200a2e2e8bc43633707d8365f73130c5bca3923b9.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO resource_type (workspace_id, name, schema, description, format_extension, edited_at)\n VALUES ('admins', $1, $2, $3, $4, now())\n ON CONFLICT (workspace_id, name) DO UPDATE\n SET schema = EXCLUDED.schema, description = EXCLUDED.description,\n -- A fileset is a set of files, so it cannot also be one file.\n -- Create and update reject the pair; this writer bypasses both, so\n -- it declines the extension rather than persisting the forbidden\n -- combination onto a same-named local fileset.\n format_extension = CASE\n WHEN resource_type.is_fileset THEN NULL\n WHEN $5 THEN EXCLUDED.format_extension\n ELSE resource_type.format_extension END,\n edited_at = now()", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Jsonb", - "Text", - "Varchar", - "Bool" - ] - }, - "nullable": [] - }, - "hash": "972df41db505fbbd20a558b200a2e2e8bc43633707d8365f73130c5bca3923b9" -} diff --git a/backend/.sqlx/query-5899c7614f195fdd23e38389e52b004f957aafa2201b80638b5f87a625373f00.json b/backend/.sqlx/query-dd6f4b505f4c1e2c734c5d04528c95bd6b3fb6ebf3ba115160573487cee4a606.json similarity index 64% rename from backend/.sqlx/query-5899c7614f195fdd23e38389e52b004f957aafa2201b80638b5f87a625373f00.json rename to backend/.sqlx/query-dd6f4b505f4c1e2c734c5d04528c95bd6b3fb6ebf3ba115160573487cee4a606.json index 400c8d9ee6..340b09ca27 100644 --- a/backend/.sqlx/query-5899c7614f195fdd23e38389e52b004f957aafa2201b80638b5f87a625373f00.json +++ b/backend/.sqlx/query-dd6f4b505f4c1e2c734c5d04528c95bd6b3fb6ebf3ba115160573487cee4a606.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO resource_type\n (workspace_id, name, schema, description, created_by, format_extension, is_fileset, edited_at)\n VALUES ($1, $2, $3, $4, $5, $6, $7, now())", + "query": "INSERT INTO resource_type\n (workspace_id, name, schema, description, created_by, format_extension, is_fileset, display_name, edited_at)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, now())", "describe": { "columns": [], "parameters": { @@ -11,10 +11,11 @@ "Text", "Varchar", "Varchar", - "Bool" + "Bool", + "Varchar" ] }, "nullable": [] }, - "hash": "5899c7614f195fdd23e38389e52b004f957aafa2201b80638b5f87a625373f00" + "hash": "dd6f4b505f4c1e2c734c5d04528c95bd6b3fb6ebf3ba115160573487cee4a606" } diff --git a/backend/migrations/20260914131148_resource_type_display_name.down.sql b/backend/migrations/20260914131148_resource_type_display_name.down.sql new file mode 100644 index 0000000000..57c9ffabc8 --- /dev/null +++ b/backend/migrations/20260914131148_resource_type_display_name.down.sql @@ -0,0 +1 @@ +ALTER TABLE resource_type DROP COLUMN display_name; diff --git a/backend/migrations/20260914131148_resource_type_display_name.up.sql b/backend/migrations/20260914131148_resource_type_display_name.up.sql new file mode 100644 index 0000000000..213622ef94 --- /dev/null +++ b/backend/migrations/20260914131148_resource_type_display_name.up.sql @@ -0,0 +1,24 @@ +-- The name a product goes by, beside the identifier a resource references: `gsheets` is +-- "Google Sheets". Null where nobody named the type; readers derive a label from the name. +ALTER TABLE resource_type ADD COLUMN display_name VARCHAR(100); + +-- The names the hub carries today, so existing instances show them before any sync. Only in +-- admins, where hub resource types live and every workspace reads them from. +UPDATE resource_type SET display_name = v.display_name +FROM (VALUES + ('bamboo_hr', 'BambooHR'), + ('cacertificate', 'CA certificate'), + ('deep_infra', 'DeepInfra'), + ('gcal', 'Google Calendar'), + ('gdocs', 'Google Docs'), + ('gdrive', 'Google Drive'), + ('gforms', 'Google Forms'), + ('gsheets', 'Google Sheets'), + ('gworkspace', 'Google Workspace'), + ('sensortower', 'Sensor Tower'), + ('snowflake_oauth', 'Snowflake (OAuth)'), + ('their_stack', 'TheirStack') +) AS v(name, display_name) +WHERE resource_type.workspace_id = 'admins' + AND resource_type.name = v.name + AND resource_type.display_name IS NULL; diff --git a/backend/src/main.rs b/backend/src/main.rs index 7f2ce3cd82..316ed099f2 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -411,6 +411,13 @@ struct HubResourceTypeRaw { /// Absent from hubs predating the column, and from caches written before it. #[serde(default)] pub format_extension: Option, + /// Doubly optional, so a hub predating the field (no key) is told apart from a type the + /// hub leaves unnamed (null). + #[serde( + default, + deserialize_with = "windmill_common::more_serde::double_option" + )] + pub display_name: Option>, } @@ -434,6 +441,14 @@ pub struct HubResourceType { skip_serializing_if = "Option::is_none" )] pub format_extension: Option>, + /// Doubly optional like `format_extension`: a cache written before the field leaves the + /// stored name alone, while a null from the hub clears it. + #[serde( + default, + deserialize_with = "windmill_common::more_serde::double_option", + skip_serializing_if = "Option::is_none" + )] + pub display_name: Option>, } const HUB_RT_CACHE_FILE: &str = "resource_types.json"; @@ -481,6 +496,7 @@ async fn cache_hub_resource_types() -> anyhow::Result<()> { app: rt.app, description: rt.description, format_extension: Some(rt.format_extension), + display_name: rt.display_name, }) }) .collect(); @@ -531,8 +547,9 @@ pub async fn sync_cached_resource_types(db: &sqlx::Pool) -> anyh Option, Option, bool, + Option, )> = sqlx::query_as( - "SELECT name, schema, description, format_extension, is_fileset FROM resource_type WHERE workspace_id = 'admins'", + "SELECT name, schema, description, format_extension, is_fileset, display_name FROM resource_type WHERE workspace_id = 'admins'", ) .fetch_all(db) .await @@ -540,12 +557,23 @@ pub async fn sync_cached_resource_types(db: &sqlx::Pool) -> anyh let existing_map: std::collections::HashMap< String, - (Option, Option, Option, bool), + ( + Option, + Option, + Option, + bool, + Option, + ), > = existing_types .into_iter() - .map(|(name, schema, desc, format_extension, is_fileset)| { - (name, (schema, desc, format_extension, is_fileset)) - }) + .map( + |(name, schema, desc, format_extension, is_fileset, display_name)| { + ( + name, + (schema, desc, format_extension, is_fileset, display_name), + ) + }, + ) .collect(); let mut synced_count = 0; @@ -553,8 +581,9 @@ pub async fn sync_cached_resource_types(db: &sqlx::Pool) -> anyh for rt in cached_types { let existing = existing_map.get(&rt.name); - let is_fileset = existing.map(|(_, _, _, f)| *f).unwrap_or(false); - let stored_extension = existing.and_then(|(_, _, e, _)| e.clone()); + let is_fileset = existing.map(|(_, _, _, f, _)| *f).unwrap_or(false); + let stored_extension = existing.and_then(|(_, _, e, _, _)| e.clone()); + let stored_display_name = existing.and_then(|(_, _, _, _, n)| n.clone()); // A fileset is a set of files, so it cannot also be one file. Create, update // and the manual sync all reject the pair; this writer would otherwise // persist it onto a same-named local fileset. @@ -570,11 +599,25 @@ pub async fn sync_cached_resource_types(db: &sqlx::Pool) -> anyh None => stored_extension.clone(), } }; + // No key in the cache leaves the stored name alone, as for the extension. So does a name + // too long for the column: one bad entry must not fail the upsert and end the sync. + let display_name = match &rt.display_name { + Some(Some(name)) if name.chars().count() > 100 => { + tracing::warn!( + "Ignoring the display_name of resource type {}: longer than 100 characters", + rt.name + ); + stored_display_name.clone() + } + Some(from_cache) => from_cache.clone(), + None => stored_display_name.clone(), + }; - if let Some((existing_schema, existing_desc, _, _)) = existing { + if let Some((existing_schema, existing_desc, _, _, _)) = existing { if existing_schema == &rt.schema && existing_desc == &rt.description && stored_extension == format_extension + && stored_display_name == display_name { skipped_count += 1; continue; @@ -586,16 +629,18 @@ pub async fn sync_cached_resource_types(db: &sqlx::Pool) -> anyh // `format_extension` is resolved above rather than coalesced here: a // COALESCE could never clear one, so a hub that dropped an extension // would leave the stale value behind forever. - "INSERT INTO resource_type (workspace_id, name, schema, description, format_extension, edited_at) - VALUES ('admins', $1, $2, $3, $4, now()) + "INSERT INTO resource_type (workspace_id, name, schema, description, format_extension, display_name, edited_at) + VALUES ('admins', $1, $2, $3, $4, $5, now()) ON CONFLICT (workspace_id, name) DO UPDATE SET schema = EXCLUDED.schema, description = EXCLUDED.description, - format_extension = EXCLUDED.format_extension, edited_at = now()", + format_extension = EXCLUDED.format_extension, + display_name = EXCLUDED.display_name, edited_at = now()", ) .bind(&rt.name) .bind(&rt.schema) .bind(&rt.description) .bind(&format_extension) + .bind(&display_name) .execute(db) .await .with_context(|| format!("Failed to upsert resource type {}", rt.name))?; diff --git a/backend/summarized_schema.txt b/backend/summarized_schema.txt index 1ab05bc584..463afd0d33 100644 --- a/backend/summarized_schema.txt +++ b/backend/summarized_schema.txt @@ -173,7 +173,7 @@ raw_app: path(char), version(int), workspace_id(char), summary(char), edited_at( FK: (workspace_id) -> workspace(id) resource: workspace_id(char), path(char), value(jsonb), description(text), resource_type(char), extra_perms(jsonb), edited_at(ts), created_by(char), labels(text[]) FK: (workspace_id) -> workspace(id) -resource_type: workspace_id(char), name(char), schema(jsonb), description(text), edited_at(ts), created_by(char), format_extension(char), is_fileset(bool) +resource_type: workspace_id(char), name(char), schema(jsonb), description(text), edited_at(ts), created_by(char), format_extension(char), is_fileset(bool), display_name(char) FK: (workspace_id) -> workspace(id) resume_job: id(uuid), job(uuid), flow(uuid), created_at(ts), value(jsonb), approver(char), resume_id(int), approved(bool) FK: (flow) -> v2_job_queue(id) diff --git a/backend/windmill-api-embeddings/src/lib.rs b/backend/windmill-api-embeddings/src/lib.rs index 35816dfaa8..1de99fb44f 100644 --- a/backend/windmill-api-embeddings/src/lib.rs +++ b/backend/windmill-api-embeddings/src/lib.rs @@ -418,7 +418,7 @@ impl EmbeddingsDb { let hub_resource_types = response.json::>().await?; let resource_types: Vec = - sqlx::query_as!(ResourceType, "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset from resource_type ORDER BY name",) + sqlx::query_as!(ResourceType, "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset, display_name from resource_type ORDER BY name",) .fetch_all(pg_db) .await?; diff --git a/backend/windmill-api-integration-tests/tests/resources.rs b/backend/windmill-api-integration-tests/tests/resources.rs index 3f0b21b184..53648ec759 100644 --- a/backend/windmill-api-integration-tests/tests/resources.rs +++ b/backend/windmill-api-integration-tests/tests/resources.rs @@ -443,6 +443,30 @@ async fn test_resource_endpoints(db: Pool) -> anyhow::Result<()> { let body = resp.json::().await?; assert_eq!(body["description"], "Updated type desc"); + // display_name: an update that omits it, as a push from a CLI predating the field does, + // keeps it; an explicit null clears it. + for (update, expected) in [ + ( + json!({"display_name": "New Test Type"}), + json!("New Test Type"), + ), + ( + json!({"description": "Updated type desc"}), + json!("New Test Type"), + ), + (json!({"display_name": null}), serde_json::Value::Null), + ] { + let resp = authed(client().post(resource_url(port, "type/update", "new_test_type"))) + .json(&update) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let resp = authed_get(port, "type/get", "new_test_type").await; + let body = resp.json::().await?; + assert_eq!(body["display_name"], expected); + } + // type/delete let resp = authed(client().delete(resource_url(port, "type/delete", "new_test_type"))) .send() diff --git a/backend/windmill-api-settings/src/lib.rs b/backend/windmill-api-settings/src/lib.rs index 0fe358da1c..2e2b3290fa 100644 --- a/backend/windmill-api-settings/src/lib.rs +++ b/backend/windmill-api-settings/src/lib.rs @@ -2074,6 +2074,13 @@ struct CachedResourceType { deserialize_with = "windmill_common::more_serde::double_option" )] format_extension: Option>, + /// Doubly optional like `format_extension`: no key leaves the stored name alone, an explicit + /// null (the hub naming nothing) clears it. + #[serde( + default, + deserialize_with = "windmill_common::more_serde::double_option" + )] + display_name: Option>, } #[derive(serde::Deserialize)] @@ -2085,6 +2092,11 @@ struct HubResourceTypeRaw { description: Option, #[serde(default)] format_extension: Option, + #[serde( + default, + deserialize_with = "windmill_common::more_serde::double_option" + )] + display_name: Option>, } async fn fetch_resource_types_from_hub() -> error::Result> { @@ -2127,6 +2139,7 @@ async fn fetch_resource_types_from_hub() -> error::Result 100 => None, + other => other.clone(), + }; let exists: Option = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM resource_type WHERE workspace_id = 'admins' AND name = $1 AND schema IS NOT DISTINCT FROM $2 AND description IS NOT DISTINCT FROM $3 AND ($5 IS NOT TRUE OR format_extension IS NOT DISTINCT FROM $4))", + "SELECT EXISTS(SELECT 1 FROM resource_type WHERE workspace_id = 'admins' AND name = $1 AND schema IS NOT DISTINCT FROM $2 AND description IS NOT DISTINCT FROM $3 AND ($5 IS NOT TRUE OR format_extension IS NOT DISTINCT FROM $4) AND ($7 IS NOT TRUE OR display_name IS NOT DISTINCT FROM $6))", &rt.name, rt.schema.as_ref(), rt.description.as_deref(), rt.format_extension.clone().flatten(), rt.format_extension.is_some(), + display_name.clone().flatten(), + display_name.is_some(), ) .fetch_one(&db) .await?; @@ -2198,8 +2219,8 @@ async fn sync_cached_resource_types( // Whether the payload carried the key at all is what decides: present // (even as null) is authoritative and may clear, absent means a cache // written before the column and must leave the stored value alone. - "INSERT INTO resource_type (workspace_id, name, schema, description, format_extension, edited_at) - VALUES ('admins', $1, $2, $3, $4, now()) + "INSERT INTO resource_type (workspace_id, name, schema, description, format_extension, display_name, edited_at) + VALUES ('admins', $1, $2, $3, $4, $6, now()) ON CONFLICT (workspace_id, name) DO UPDATE SET schema = EXCLUDED.schema, description = EXCLUDED.description, -- A fileset is a set of files, so it cannot also be one file. @@ -2210,12 +2231,15 @@ async fn sync_cached_resource_types( WHEN resource_type.is_fileset THEN NULL WHEN $5 THEN EXCLUDED.format_extension ELSE resource_type.format_extension END, + display_name = CASE WHEN $7 THEN EXCLUDED.display_name ELSE resource_type.display_name END, edited_at = now()", &rt.name, rt.schema.as_ref(), rt.description.as_deref(), rt.format_extension.clone().flatten(), rt.format_extension.is_some(), + display_name.clone().flatten(), + display_name.is_some(), ) .execute(&db) .await?; diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 640c0fe06a..6048772ff6 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -6491,8 +6491,8 @@ async fn clone_resource_types( target_workspace_id: &str, ) -> Result<()> { sqlx::query!( - "INSERT INTO resource_type (workspace_id, name, schema, description, edited_at, created_by, format_extension, is_fileset) - SELECT $2, name, schema, description, edited_at, created_by, format_extension, is_fileset + "INSERT INTO resource_type (workspace_id, name, schema, description, edited_at, created_by, format_extension, is_fileset, display_name) + SELECT $2, name, schema, description, edited_at, created_by, format_extension, is_fileset, display_name FROM resource_type WHERE workspace_id = $1", source_workspace_id, diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index b733157d85..21565ffaf7 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -9280,6 +9280,10 @@ paths: picks: description: how often the integration has been picked, absent on a hub that does not count picks type: integer + display_name: + description: the label the hub curates for the integration, null or absent where it names none + type: string + nullable: true required: - name @@ -30777,6 +30781,11 @@ components: type: string is_fileset: type: boolean + display_name: + type: string + description: >- + The name the product goes by, e.g. "Google Sheets" for gsheets. + Absent where nobody named the type. required: - name @@ -30794,6 +30803,12 @@ components: description: >- File extension for a type whose value is one file rather than a set of fields. Omit to leave it unchanged; send null to clear it. + display_name: + type: string + nullable: true + description: >- + The name the product goes by. Omit to leave it unchanged; send null + to clear it. TriggerHistoryEntry: type: object diff --git a/backend/windmill-api/src/workspaces_export.rs b/backend/windmill-api/src/workspaces_export.rs index a85f9d004e..69565ffcbd 100644 --- a/backend/windmill-api/src/workspaces_export.rs +++ b/backend/windmill-api/src/workspaces_export.rs @@ -924,7 +924,7 @@ pub(crate) async fn tarball_workspace( if !skip_resource_types.unwrap_or(false) { let resource_types = sqlx::query_as!( ResourceType, - "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset FROM resource_type WHERE workspace_id = $1", + "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset, display_name FROM resource_type WHERE workspace_id = $1", &w_id ) .fetch_all(&mut *tx) diff --git a/backend/windmill-store/src/resources.rs b/backend/windmill-store/src/resources.rs index 126cf97106..5bb15b456e 100644 --- a/backend/windmill-store/src/resources.rs +++ b/backend/windmill-store/src/resources.rs @@ -118,6 +118,10 @@ pub struct ResourceType { pub edited_at: Option>, pub format_extension: Option, pub is_fileset: bool, + /// The name the product goes by (`gsheets` is "Google Sheets"), null where nobody named it. + /// Skipped when absent, so the type files of a synced repo gain nothing until one is set. + #[serde(skip_serializing_if = "Option::is_none")] + pub display_name: Option, } #[derive(Deserialize)] @@ -127,6 +131,7 @@ pub struct CreateResourceType { pub description: Option, pub format_extension: Option, pub is_fileset: Option, + pub display_name: Option, } #[derive(Deserialize)] @@ -143,6 +148,13 @@ pub struct EditResourceType { deserialize_with = "windmill_common::more_serde::double_option" )] pub format_extension: Option>, + /// Doubly optional for the same reason. A push from a CLI that predates the field omits it, + /// and must not clear a name the hub set. + #[serde( + default, + deserialize_with = "windmill_common::more_serde::double_option" + )] + pub display_name: Option>, } #[derive(FromRow, Serialize, Deserialize)] @@ -2729,7 +2741,7 @@ async fn list_resource_types( ) -> JsonResult> { let rows = sqlx::query_as!( ResourceType, - "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset from resource_type WHERE (workspace_id = $1 OR workspace_id = 'admins') ORDER \ + "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset, display_name from resource_type WHERE (workspace_id = $1 OR workspace_id = 'admins') ORDER \ BY name", &w_id ) @@ -3109,7 +3121,7 @@ async fn get_resource_type( let resource_type_o = sqlx::query_as!( ResourceType, - "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset from resource_type WHERE name = $1 AND (workspace_id = $2 OR workspace_id = 'admins')", + "SELECT workspace_id, name, schema, description, created_by, edited_at, format_extension, is_fileset, display_name from resource_type WHERE name = $1 AND (workspace_id = $2 OR workspace_id = 'admins')", &name, &w_id ) @@ -3137,6 +3149,20 @@ async fn exists_resource_type( Ok(Json(exists)) } +/// Trimmed, blank as none, and held to the column's 100 characters, so an over-long name is +/// refused with a message rather than a database error. +fn normalize_display_name(name: Option<&str>) -> Result> { + let Some(name) = name.map(str::trim).filter(|n| !n.is_empty()) else { + return Ok(None); + }; + if name.chars().count() > 100 { + return Err(Error::BadRequest( + "display_name must be at most 100 characters".to_string(), + )); + } + Ok(Some(name.to_string())) +} + async fn create_resource_type( authed: ApiAuthed, Extension(db): Extension, @@ -3170,11 +3196,12 @@ async fn create_resource_type( "A fileset resource type cannot have a format_extension".to_string(), )); } + let display_name = normalize_display_name(resource_type.display_name.as_deref())?; sqlx::query!( "INSERT INTO resource_type - (workspace_id, name, schema, description, created_by, format_extension, is_fileset, edited_at) - VALUES ($1, $2, $3, $4, $5, $6, $7, now())", + (workspace_id, name, schema, description, created_by, format_extension, is_fileset, display_name, edited_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, now())", w_id, resource_type.name, resource_type.schema, @@ -3182,6 +3209,7 @@ async fn create_resource_type( authed.username, resource_type.format_extension, is_fileset, + display_name, ) .execute(&mut *tx) .await?; @@ -3349,6 +3377,12 @@ async fn update_resource_type( None => sqlb.set("format_extension", "NULL"), }; } + if let Some(display_name) = &ns.display_name { + match normalize_display_name(display_name.as_deref())? { + Some(name) => sqlb.set_str("display_name", name), + None => sqlb.set("display_name", "NULL"), + }; + } sqlb.set_str("edited_at", "now()"); let sql = sqlb.sql().map_err(|e| Error::internal_err(e.to_string()))?; let mut tx = user_db.begin(&authed).await?; diff --git a/cli/src/commands/hub/hub.ts b/cli/src/commands/hub/hub.ts index f752147811..1a9c89680e 100644 --- a/cli/src/commands/hub/hub.ts +++ b/cli/src/commands/hub/hub.ts @@ -20,6 +20,9 @@ interface HubResourceType { // Absent from hubs predating the column, so a missing value is "ordinary type", // not "unset it". format_extension?: string | null; + // Null where nobody named the type, and absent from hubs predating the field, which + // leaves a stored name alone rather than clearing it. + display_name?: string | null; } export async function pull(opts: GlobalOptions) { @@ -120,7 +123,9 @@ export async function pull(opts: GlobalOptions) { deepEqual(y.schema, x.schema) && y.description === x.description && (y.is_fileset ?? false) === (x.is_fileset ?? false) && - (y.format_extension ?? null) === (x.format_extension ?? null) + (y.format_extension ?? null) === (x.format_extension ?? null) && + (x.display_name === undefined || + (y.display_name ?? null) === x.display_name) ) ) { log.info("skipping " + x.name + " (same as current)"); diff --git a/cli/src/commands/resource-type/resource-type.ts b/cli/src/commands/resource-type/resource-type.ts index fd4b72108e..505200f07a 100644 --- a/cli/src/commands/resource-type/resource-type.ts +++ b/cli/src/commands/resource-type/resource-type.ts @@ -28,6 +28,7 @@ export interface ResourceTypeFile { // Extension for a type whose value is one file rather than a set of fields; it // is what makes the resource editor a file editor for that language. format_extension?: string | null; + display_name?: string | null; } export async function pushResourceType( diff --git a/frontend/src/lib/components/AppConnectInner.svelte b/frontend/src/lib/components/AppConnectInner.svelte index 99adeac6b3..cef58fcdb2 100644 --- a/frontend/src/lib/components/AppConnectInner.svelte +++ b/frontend/src/lib/components/AppConnectInner.svelte @@ -5,10 +5,12 @@ import LabelsInput from './LabelsInput.svelte' import IconedResourceType from './IconedResourceType.svelte' import { + addResourceTypeDisplayName, isCustomResourceTypeName, resourceTypeDisplayName, resourceTypeMatchRank, resourceTypeSearchText, + setResourceTypeDisplayNames, sortResourceTypesByMatch } from './resourceTypeDisplay' import { @@ -497,6 +499,7 @@ // $derived, so search re-ranks when they land. ResourceService.listResourceType({ workspace: effectiveWorkspace }) .then((types) => { + setResourceTypeDisplayNames(types) resourceTypeDescriptions = Object.fromEntries( types.filter((t) => t.description).map((t) => [t.name, t.description!]) ) @@ -652,6 +655,7 @@ workspace: effectiveWorkspace, path: resourceType }) + addResourceTypeDisplayName(resourceTypeInfo) const props: Record = resourceTypeInfo?.schema?.['properties'] ?? {} const newArgsKeys = Object.keys(props).filter((x) => props?.[x]?.type == 'string') ?? [] diff --git a/frontend/src/lib/components/ImportSetupStep.svelte b/frontend/src/lib/components/ImportSetupStep.svelte index 51eaf725f8..f85ebf7395 100644 --- a/frontend/src/lib/components/ImportSetupStep.svelte +++ b/frontend/src/lib/components/ImportSetupStep.svelte @@ -19,7 +19,11 @@ import { applyRetarget, seesWholeWorkspace } from '$lib/importWizard/retargetDeployed' import { OauthService } from '$lib/gen' import { registryCcCapableFor } from '$lib/components/oauthRegistry' - import { resourceTypeDisplayName } from '$lib/components/resourceTypeDisplay' + import { + addResourceTypeDisplayName, + resourceTypeDisplayName + } from '$lib/components/resourceTypeDisplay' + import { loadResourceTypeDisplayName } from '$lib/components/displayNameLoaders' import { applyOneMigration } from '$lib/components/workspaceSettings/projectInstall' import { probeMigrationsApplied } from '$lib/importWizard/probe' import { @@ -197,6 +201,14 @@ * first half and Connect disappears on the eight such providers, where it would work. */ const canConnectType = (rt: string) => instanceConnects.has(rt) || registryCcCapableFor(rt) + + // A row blocked by a resource of another type names that type, whose row nothing here reads. + $effect(() => { + for (const b of blanks) { + if (b.occupiedBy) void loadResourceTypeDisplayName(workspace, b.occupiedBy) + } + }) + let appConnect: AppConnectDrawer | undefined = $state(undefined) const customInstanceDbs = resource([() => workspace], SettingService.listCustomInstanceDbs) @@ -396,8 +408,9 @@ // row is kept instead; it just cannot name which fields are short. let requirementsUnknown = false try { - const schema = (await ResourceService.getResourceType({ workspace, path: r.resource_type })) - ?.schema as { required?: string[] } | undefined + const rt = await ResourceService.getResourceType({ workspace, path: r.resource_type }) + addResourceTypeDisplayName(rt) + const schema = rt?.schema as { required?: string[] } | undefined required = schema?.required ?? [] } catch { requirementsUnknown = true diff --git a/frontend/src/lib/components/LightweightResourcePicker.svelte b/frontend/src/lib/components/LightweightResourcePicker.svelte index bb0c21b4c1..58557e8c8b 100644 --- a/frontend/src/lib/components/LightweightResourcePicker.svelte +++ b/frontend/src/lib/components/LightweightResourcePicker.svelte @@ -10,6 +10,7 @@ import Select from './select/Select.svelte' import IconedResourceType from './IconedResourceType.svelte' import { addResourceTitle } from './resourceTypeDisplay' + import { loadResourceTypeDisplayName } from './displayNameLoaders' interface Props { value: string | undefined @@ -218,6 +219,7 @@ on:click={() => { refreshCount += 1 open = true + if (ws && resourceType) void loadResourceTypeDisplayName(ws, resourceType) drawer?.openDrawer?.() }} startIcon={{ icon: Plus }} diff --git a/frontend/src/lib/components/ResourceEditorDrawer.svelte b/frontend/src/lib/components/ResourceEditorDrawer.svelte index 36d49d9149..052c8c9e92 100644 --- a/frontend/src/lib/components/ResourceEditorDrawer.svelte +++ b/frontend/src/lib/components/ResourceEditorDrawer.svelte @@ -19,6 +19,7 @@ import ResourceVersionHistory from './ResourceVersionHistory.svelte' import IconedResourceType from './IconedResourceType.svelte' import { addResourceTitle } from './resourceTypeDisplay' + import { loadResourceTypeDisplayName } from './displayNameLoaders' let { workspace = undefined, @@ -111,6 +112,8 @@ // rather than left where the last one put it: a new resource is a typed form, whoever was // looking at JSON before. viewJsonSchema = false + // The title names the type, whose row nothing else on the page may have read. + void loadResourceTypeDisplayName(effectiveWorkspace, resourceType) drawer?.openDrawer?.() } diff --git a/frontend/src/lib/components/ResourceTypePicker.svelte b/frontend/src/lib/components/ResourceTypePicker.svelte index 53dc0ed7a2..35d6593635 100644 --- a/frontend/src/lib/components/ResourceTypePicker.svelte +++ b/frontend/src/lib/components/ResourceTypePicker.svelte @@ -9,7 +9,11 @@ import Tooltip from './Tooltip.svelte' import Badge from './common/badge/Badge.svelte' import { untrack } from 'svelte' - import { resourceTypeSearchText, sortResourceTypesByMatch } from './resourceTypeDisplay' + import { + resourceTypeSearchText, + setResourceTypeDisplayNames, + sortResourceTypesByMatch + } from './resourceTypeDisplay' interface Props { value: string | undefined notPickable?: boolean @@ -22,6 +26,7 @@ async function loadResources() { const types = await ResourceService.listResourceType({ workspace: $workspaceStore! }) + setResourceTypeDisplayNames(types) resources = types.map((t) => ({ name: t.name, description: t.description, diff --git a/frontend/src/lib/components/displayNameLoaders.ts b/frontend/src/lib/components/displayNameLoaders.ts new file mode 100644 index 0000000000..5eed1e81a0 --- /dev/null +++ b/frontend/src/lib/components/displayNameLoaders.ts @@ -0,0 +1,71 @@ +import { get } from 'svelte/store' +import { IntegrationService, ResourceService } from '$lib/gen' +import { disableHubStore } from '$lib/stores' +import { createCache } from '$lib/utils' +import { addResourceTypeDisplayName, setHubIntegrationDisplayNames } from './resourceTypeDisplay' + +/** + * Loads what `resourceTypeDisplayName` and `integrationDisplayName` read: a type's stored name for a + * surface that holds no row for it, and the hub's integration list, which every picker that needs + * it shares. Apart from `resourceTypeDisplay`, which makes no API calls so it can be unit-tested + * alone. Cached briefly: drawers and pickers reopen often, and a name rarely changes. + */ +const CACHE_MS = 60_000 + +const resourceTypeRowCached = createCache( + ({ workspace, name }: { workspace: string; name: string }) => + ResourceService.getResourceType({ workspace, path: name }).then( + (rt) => addResourceTypeDisplayName(rt), + () => {} + ), + { invalidateMs: CACHE_MS, maxSize: 50 } +) + +/** + * Fill `resourceTypeDisplayName` for one type, for a surface titled with a type it holds no row + * for. The name is stored with the type, so this reads the row rather than the hub. + */ +export function loadResourceTypeDisplayName(workspace: string, name: string): Promise { + return resourceTypeRowCached({ workspace, name }) +} + +/** Bumped by every failed read, so the next caller keys a fresh read rather than the rejection. */ +let failedReads = 0 + +const hubIntegrationsCached = createCache( + ({ kind }: { kind?: string; refresh: number; attempt: number }) => + IntegrationService.listHubIntegrations({ kind }).then( + (integrations) => { + setHubIntegrationDisplayNames(integrations) + return integrations + }, + (error) => { + failedReads += 1 + throw error + } + ), + { invalidateMs: CACHE_MS } +) + +/** + * The hub's integration list, read once a minute per `kind` however many pickers ask, recording + * each integration's name on the way. A failed read rejects, so a picker can say the hub is + * unavailable, but is not kept: the next caller reads again. `refresh` is a picker's refresh + * count, and a new value reads again too. + */ +export function listHubIntegrationsShared(kind?: string, refresh = 0) { + return hubIntegrationsCached({ kind, refresh, attempt: failedReads }) +} + +/** + * Fill `integrationDisplayName` for a picker whose integrations come from its own items rather + * than the hub's integration list, as the hub app and flow pickers do. Unfiltered: `kind` + * narrows by script kind, so asking for an app or a flow would name nothing. + */ +export function loadHubIntegrationDisplayNames(): Promise { + if (get(disableHubStore)) return Promise.resolve() + return listHubIntegrationsShared().then( + () => {}, + () => {} + ) +} diff --git a/frontend/src/lib/components/flows/pickers/PickHubApp.svelte b/frontend/src/lib/components/flows/pickers/PickHubApp.svelte index 234f053f7b..15177d6cd6 100644 --- a/frontend/src/lib/components/flows/pickers/PickHubApp.svelte +++ b/frontend/src/lib/components/flows/pickers/PickHubApp.svelte @@ -6,6 +6,7 @@ import NoItemFound from '$lib/components/home/NoItemFound.svelte' import RowIcon from '$lib/components/common/table/RowIcon.svelte' import { loadHubApps } from '$lib/hub' + import { loadHubIntegrationDisplayNames } from '$lib/components/displayNameLoaders' import TextInput from '$lib/components/text_input/TextInput.svelte' import { Alert } from '$lib/components/common' import { disableHubStore } from '$lib/stores' @@ -36,6 +37,7 @@ onMount(async () => { if ($disableHubStore) return + void loadHubIntegrationDisplayNames() const result = await loadHubApps() if (result === undefined) { hubNotAvailable = true diff --git a/frontend/src/lib/components/flows/pickers/PickHubFlow.svelte b/frontend/src/lib/components/flows/pickers/PickHubFlow.svelte index 1f3c659bf8..8c07ad1506 100644 --- a/frontend/src/lib/components/flows/pickers/PickHubFlow.svelte +++ b/frontend/src/lib/components/flows/pickers/PickHubFlow.svelte @@ -6,6 +6,7 @@ import NoItemFound from '$lib/components/home/NoItemFound.svelte' import RowIcon from '$lib/components/common/table/RowIcon.svelte' import { loadHubFlows } from '$lib/hub' + import { loadHubIntegrationDisplayNames } from '$lib/components/displayNameLoaders' import TextInput from '$lib/components/text_input/TextInput.svelte' import { Alert } from '$lib/components/common' import { disableHubStore } from '$lib/stores' @@ -36,6 +37,7 @@ onMount(async () => { if ($disableHubStore) return + void loadHubIntegrationDisplayNames() const result = await loadHubFlows() if (result === undefined) { hubNotAvailable = true diff --git a/frontend/src/lib/components/flows/pickers/PickHubScript.svelte b/frontend/src/lib/components/flows/pickers/PickHubScript.svelte index d84228dd35..05205d8335 100644 --- a/frontend/src/lib/components/flows/pickers/PickHubScript.svelte +++ b/frontend/src/lib/components/flows/pickers/PickHubScript.svelte @@ -4,8 +4,9 @@ import { capitalize } from '$lib/utils' import NoItemFound from '$lib/components/home/NoItemFound.svelte' import { APP_TO_ICON_COMPONENT } from '$lib/components/icons' + import { listHubIntegrationsShared } from '$lib/components/displayNameLoaders' import ListFilters from '$lib/components/home/ListFilters.svelte' - import { IntegrationService, ScriptService, type HubScriptKind } from '$lib/gen' + import { ScriptService, type HubScriptKind } from '$lib/gen' import { Loader2 } from 'lucide-svelte' import TextInput from '$lib/components/text_input/TextInput.svelte' import { disableHubStore, workspaceStore } from '$lib/stores' @@ -64,7 +65,7 @@ hubNotAvailable = false // Independent reads, so they share one round trip before first paint. const [integrations, local] = await Promise.all([ - IntegrationService.listHubIntegrations({ kind: filterKind }), + listHubIntegrationsShared(filterKind), $workspaceStore ? localCountsByIntegration($workspaceStore) : {} ]) const hubPicks = Object.fromEntries(integrations.map((x) => [x.name, x.picks ?? 0])) diff --git a/frontend/src/lib/components/flows/pickers/PickHubScriptQuick.svelte b/frontend/src/lib/components/flows/pickers/PickHubScriptQuick.svelte index d32e82c702..2499c285c4 100644 --- a/frontend/src/lib/components/flows/pickers/PickHubScriptQuick.svelte +++ b/frontend/src/lib/components/flows/pickers/PickHubScriptQuick.svelte @@ -1,9 +1,6 @@
    - {#if !hideSidebar} - + {#if chat && chatState} + {#if !hideSidebar} + + {/if} + {/if} -
    diff --git a/frontend/src/lib/components/flows/conversations/FlowChatInterface.svelte b/frontend/src/lib/components/flows/conversations/FlowChatInterface.svelte index ab52d06102..22fe037728 100644 --- a/frontend/src/lib/components/flows/conversations/FlowChatInterface.svelte +++ b/frontend/src/lib/components/flows/conversations/FlowChatInterface.svelte @@ -3,19 +3,76 @@ import { MessageCircle, Loader2, Settings2 } from 'lucide-svelte' import ChatMessage from '$lib/components/chat/ChatMessage.svelte' import ChatInput from '$lib/components/chat/ChatInput.svelte' - import { FlowChatManager } from './FlowChatManager.svelte' import Modal from '$lib/components/common/modal/Modal.svelte' import SchemaForm from '$lib/components/SchemaForm.svelte' import { type DynamicInput } from '$lib/utils' + import { tick, untrack } from 'svelte' + import type { Chat, ChatState } from 'windmill-chat' interface Props { - manager: FlowChatManager + chat: Chat + chatState: ChatState deploymentInProgress?: boolean additionalInputsSchema?: Record path: string + workspace?: string } - let { manager, deploymentInProgress = false, additionalInputsSchema, path }: Props = $props() + let { + chat, + chatState, + deploymentInProgress = false, + additionalInputsSchema, + path, + workspace = undefined + }: Props = $props() + + let inputMessage = $state('') + let inputElement = $state(undefined) + let messagesContainer = $state(undefined) + let loadingOlder = false + + const busy = $derived(chatState.status === 'submitted' || chatState.status === 'streaming') + // Deriveds notify only when their value changes; `chatState` itself is a new + // object on every token, and following it would drag a reader who scrolled up + // back to the end on each one. + const messageCount = $derived(chatState.messages.length) + const conversationId = $derived(chatState.conversationId) + const loadingMessages = $derived(chatState.loadingMessages) + + // Follow the conversation: new messages and a conversation switch scroll to the + // end, older pages loaded at the top keep the viewport where it was. + $effect(() => { + messageCount + conversationId + loadingMessages + untrack(() => { + if (loadingOlder) return + tick().then(() => { + if (messagesContainer) messagesContainer.scrollTop = messagesContainer.scrollHeight + }) + }) + }) + + async function handleScroll() { + if ( + !messagesContainer || + !chatState.hasMoreMessages || + chatState.loadingMessages || + loadingOlder + ) + return + if (messagesContainer.scrollTop > 10) return + loadingOlder = true + const previousHeight = messagesContainer.scrollHeight + try { + await chat.loadOlderMessages() + await tick() + messagesContainer.scrollTop = messagesContainer.scrollHeight - previousHeight + } finally { + loadingOlder = false + } + } // Derive helperScript for dynamic inputs from schema const dynamicInputHelperScript = $derived.by((): DynamicInput.HelperScript | undefined => { @@ -63,11 +120,17 @@ showInputsModal = false } - function handleSendMessage() { + async function handleSendMessage() { + const text = inputMessage.trim() + if (!text || busy || deploymentInProgress) return const inputs = additionalInputsSchema ? (loadInputsFromStorage() ?? additionalInputsValues) : undefined - manager.sendMessage(inputs) + inputMessage = '' + // A failure is reported through the chat's `onError` and as a failed message. + await chat.sendMessage(text, { inputs }).catch(() => {}) + await tick() + inputElement?.focus() } function openInputsModal() { @@ -93,7 +156,7 @@ schema={additionalInputsSchema} bind:args={additionalInputsValues} helperScript={dynamicInputHelperScript} - workspace={manager.operatingWorkspace?.()} + {workspace} /> {#snippet actions()} @@ -104,18 +167,18 @@
    {#if deploymentInProgress} {/if} - {#if manager.isLoadingMessages} + {#if chatState.loadingMessages && chatState.messages.length === 0}
    - {:else if manager.messages.length === 0} + {:else if chatState.messages.length === 0}

    Start a conversation

    @@ -123,16 +186,15 @@
    {:else}
    - {#each manager.messages as message (message.id)} + {#each chatState.messages as message (message.id)} {/each} - {#if manager.isWaitingForResponse} + {#if busy}
    Processing... @@ -148,7 +210,7 @@
    diff --git a/frontend/src/lib/components/flows/conversations/FlowChatManager.svelte.ts b/frontend/src/lib/components/flows/conversations/FlowChatManager.svelte.ts deleted file mode 100644 index bc8033e45d..0000000000 --- a/frontend/src/lib/components/flows/conversations/FlowChatManager.svelte.ts +++ /dev/null @@ -1,713 +0,0 @@ -import type { FlowConversation, FlowConversationMessage } from '$lib/gen/types.gen' -import { FlowConversationsService, JobService } from '$lib/gen' -import { sendUserToast } from '$lib/toast' -import { waitJob } from '$lib/components/waitJob' -import { tick } from 'svelte' -import InfiniteList from '$lib/components/InfiniteList.svelte' -import { workspaceStore, userStore } from '$lib/stores' -import { get } from 'svelte/store' -import { parseStreamDeltas } from '$lib/components/chat/utils' -import { randomUUID } from '$lib/utils/uuid' - -export interface ChatMessage extends FlowConversationMessage { - loading?: boolean - streaming?: boolean -} - -export interface ConversationWithDraft extends FlowConversation { - isDraft?: boolean -} - -// Per-turn stream state, kept across SSE reconnects to the same job. -interface StreamTurnState { - accumulatedContent: string - assistantMessageId: string - // Last offset the server reported; sent back on reconnect so the stream resumes - // after the deltas already rendered rather than replaying from the start. - // It indexes the stream of `streamJobId` only. - streamOffset: number | undefined - streamJobId: string | undefined -} - -export class FlowChatManager { - // State - messages = $state([]) - inputMessage = $state('') - isLoading = $state(false) - isLoadingMessages = $state(false) - isWaitingForResponse = $state(false) - messagesContainer = $state(undefined) - inputElement = $state(undefined) - page = $state(1) - hasMoreMessages = $state(false) - loadingMoreMessages = $state(false) - currentEventSource = $state(undefined) - pollingInterval = $state | undefined>(undefined) - currentJobId = $state(undefined) - conversations = $state([]) - deletingConversationId = $state(undefined) - isSidebarExpanded = $state(false) - selectedConversationId = $state(undefined) - conversationListComponent = $state(undefined) - - // Private state - #conversationsCache = $state>({}) - #scrollTimeout: ReturnType | undefined = undefined - #perPage = 50 - - // Options - #onRunFlow?: ( - userMessage: string, - conversationId: string, - additionalInputs?: Record - ) => Promise - #useStreaming = $state(false) - #path = $state(undefined) - - // When the flow editor runs as an AI-session live editor, it acts on a workspace - // that can differ from the nav store. FlowChat.svelte wires this to - // FlowEditorContext.opWorkspace so workspace-scoped calls hit the acting workspace. - operatingWorkspace?: () => string | undefined - - #workspace(): string | undefined { - return this.operatingWorkspace?.() ?? get(workspaceStore) - } - - initialize( - onRunFlow: ( - userMessage: string, - conversationId: string, - additionalInputs?: Record - ) => Promise, - path: string, - useStreaming: boolean = false - ) { - this.#onRunFlow = onRunFlow - this.#path = path - this.#useStreaming = useStreaming - } - - updateConversationId(conversationId: string | undefined) { - this.selectedConversationId = conversationId - } - - cleanup() { - if (this.currentEventSource) { - this.currentEventSource.close() - this.currentEventSource = undefined - } - this.stopPolling() - this.isLoading = false - this.isWaitingForResponse = false - this.currentJobId = undefined - } - - // Public methods for component to call - fillInputMessage(message: string) { - this.inputMessage = message - } - - focusInput() { - this.inputElement?.focus() - } - - clearMessages() { - this.messages = [] - this.inputMessage = '' - this.page = 1 - } - - async createConversation({ clearMessages = true }: { clearMessages?: boolean }) { - // Check if there's already a draft conversation - const existingDraft = this.conversations.find((c) => c.isDraft) - if (existingDraft) { - // Select the existing draft instead of creating a new one - this.selectedConversationId = existingDraft.id - this.clearMessages() - return existingDraft.id - } - const newConversationId = randomUUID() - this.selectedConversationId = newConversationId - - // Create a new conversation object and add it to the top of the list - const newConversation: ConversationWithDraft = { - id: newConversationId, - workspace_id: this.#workspace()!, - flow_path: this.#path!, - title: 'New chat', - created_at: new Date().toISOString(), - updated_at: new Date().toISOString(), - created_by: get(userStore)!.username!, - isDraft: true - } - - // Prepend to conversations list - this.conversations = [newConversation, ...this.conversations] - // Clear messages in the chat interface - if (clearMessages) { - this.clearMessages() - } - this.focusInput() - - return newConversationId - } - - setupInfiniteList() { - this.conversationListComponent?.setLoader((page, perPage) => - this.loadConversations(page, perPage) - ) - this.conversationListComponent?.setDeleteItemFn((id) => this.deleteConversation(id)) - } - - async selectConversation(conversationId: string, isDraft?: boolean) { - this.selectedConversationId = conversationId - // Load conversation messages into chat interface - if (isDraft) { - // For draft conversations, just clear messages (don't try to load from backend) - this.clearMessages() - } else { - // For persisted conversations, load messages from backend - await this.loadConversationMessages(conversationId) - } - } - - async refreshConversations() { - await this.conversationListComponent?.loadData('forceRefresh') - } - - // Only used by InfiniteList - private async deleteConversation(conversationId: string) { - try { - this.deletingConversationId = conversationId - await FlowConversationsService.deleteFlowConversation({ - workspace: this.#workspace()!, - conversationId - }) - if (this.selectedConversationId === conversationId) { - this.selectedConversationId = undefined - this.clearMessages() - } - sendUserToast('Conversation deleted successfully') - } catch (error) { - console.error('Failed to delete conversation:', error) - sendUserToast('Failed to delete conversation', true) - throw error - } finally { - this.deletingConversationId = undefined - } - } - - async cancelCurrentJob() { - if (!this.#workspace()) { - return - } - - try { - if (this.currentJobId) { - await JobService.cancelQueuedJob({ - workspace: this.#workspace()!, - id: this.currentJobId, - requestBody: {} - }) - sendUserToast(`Job ${this.currentJobId} cancelled`) - } - } catch (error) { - console.error('Error cancelling job:', error) - sendUserToast('Could not cancel job', true) - } finally { - this.cleanup() - } - } - - async loadConversationMessages(conversationId?: string) { - this.page = 1 - await this.loadMessages(true, conversationId) - } - - // Only used by InfiniteList - private async loadConversations(page: number, perPage: number) { - if (!this.#workspace() || !this.#path) return [] - - try { - const response = await FlowConversationsService.listFlowConversations({ - workspace: this.#workspace()!, - flowPath: this.#path, - page: page, - perPage: perPage - }) - - return response - } catch (error) { - console.error('Failed to load conversations:', error) - sendUserToast('Failed to load conversations', true) - return [] - } - } - - // Message loading - private async loadMessages(reset: boolean, conversationId?: string) { - let conversationIdToUse = conversationId ?? this.selectedConversationId - if (!this.#workspace() || !conversationIdToUse) return - - if (reset) { - if (this.#conversationsCache[conversationIdToUse]) { - this.messages = this.#conversationsCache[conversationIdToUse] - return - } - this.isLoadingMessages = true - } else { - this.loadingMoreMessages = true - } - - const pageToFetch = reset ? 1 : this.page + 1 - - try { - const previousScrollHeight = this.messagesContainer?.scrollHeight || 0 - - const response = await FlowConversationsService.listConversationMessages({ - workspace: this.#workspace()!, - conversationId: conversationIdToUse, - page: pageToFetch, - perPage: this.#perPage - }) - - if (reset) { - this.#conversationsCache[conversationIdToUse] = response - this.messages = response - this.isLoadingMessages = false - await new Promise((resolve) => setTimeout(resolve, 100)) - this.scrollToBottom() - } else { - this.messages = [...response, ...this.messages] - this.page = pageToFetch - // Restore scroll position - await new Promise((resolve) => setTimeout(resolve, 50)) - if (this.messagesContainer) { - this.messagesContainer.scrollTop = - this.messagesContainer.scrollHeight - previousScrollHeight - } - } - - this.hasMoreMessages = response.length === this.#perPage - } catch (error) { - console.error('Failed to load messages:', error) - sendUserToast('Failed to load messages: ' + error) - } finally { - this.isLoadingMessages = false - this.loadingMoreMessages = false - } - } - - handleScroll = () => { - if (this.#scrollTimeout) clearTimeout(this.#scrollTimeout) - - this.#scrollTimeout = setTimeout(() => { - if (!this.messagesContainer || !this.hasMoreMessages || this.loadingMoreMessages) return - - if (this.messagesContainer.scrollTop <= 10) { - this.loadMessages(false) - } - }, 200) - } - - scrollToBottom() { - if (this.messagesContainer) { - this.messagesContainer.scrollTop = this.messagesContainer.scrollHeight - } - } - - private scrollToUserMessage(messageId: string) { - if (!this.messagesContainer) return - const messageElement = this.messagesContainer.querySelector(`[data-message-id="${messageId}"]`) - if (messageElement) { - messageElement.scrollIntoView({ behavior: 'smooth', block: 'start' }) - } - } - - private getLastPersistedMessageSeq() { - for (let i = this.messages.length - 1; i >= 0; i--) { - const message = this.messages[i] - if (!message.id.startsWith('temp-')) { - return message.created_seq - } - } - - return undefined - } - - // Polling - private async pollJobResult(jobId: string) { - try { - await waitJob(jobId, this.#workspace()) - } catch (error) { - console.error('Error polling job result:', error) - } finally { - // Do a final poll to get all messages from database - try { - if (this.selectedConversationId) { - await this.pollConversationMessages(this.selectedConversationId, { - removeTempMessages: true - }) - } - } catch {} - this.cleanup() - } - } - - private async pollConversationMessages( - conversationId: string, - options?: { isNewConversation?: boolean; removeTempMessages?: boolean } - ) { - if (!this.#workspace()) return - - try { - const lastSeq = this.getLastPersistedMessageSeq() - const response = await FlowConversationsService.listConversationMessages({ - workspace: this.#workspace()!, - conversationId: conversationId, - page: 1, - perPage: 50, - afterSeq: lastSeq - }) - - if (options?.isNewConversation) { - await this.refreshConversations() - } - - const filteredResponse = response.filter((msg) => msg.message_type !== 'user') - for (const msg of filteredResponse) { - if (!this.messages.find((m) => m.id === msg.id)) { - this.messages = [...this.messages, msg] - } - } - - // Only remove temporary messages when explicitly requested (e.g., after job completion) - // During streaming, we keep temp messages to avoid them disappearing due to race conditions - if (options?.removeTempMessages) { - this.messages = this.messages.filter( - (msg) => !msg.id.startsWith('temp-') || msg.message_type === 'user' - ) - } - } catch (error) { - console.error('Polling error:', error) - } - } - - private startPolling(conversationId: string, isNewConversation?: boolean) { - if (this.pollingInterval) return - this.pollingInterval = setInterval(() => { - this.pollConversationMessages(conversationId, { isNewConversation }) - }, 500) // Poll every 0.5 seconds - setTimeout( - () => { - this.stopPolling() - }, - 2 * 60 * 1000 - ) // Stop polling after 2 minutes - } - - private stopPolling() { - if (this.pollingInterval) { - clearInterval(this.pollingInterval) - this.pollingInterval = undefined - } - } - - // Message sending - async sendMessage(additionalInputs?: Record) { - if (!this.inputMessage.trim() || this.isLoading) return - - const isNewConversation = this.messages.length === 0 - - // Reset state for new message - this.stopPolling() - - // Generate a new conversation ID if we don't have one - let currentConversationId = this.selectedConversationId - if (!this.selectedConversationId) { - const newConversationId = await this.createConversation({ clearMessages: false }) - currentConversationId = newConversationId - } - - if (!currentConversationId) { - console.error('No conversation ID found') - return - } - - // Invalidate the conversation cache - delete this.#conversationsCache[currentConversationId] - - const userMessage: ChatMessage = { - id: `temp-${randomUUID()}`, - content: this.inputMessage.trim(), - created_at: new Date().toISOString(), - created_seq: 0, - message_type: 'user', - conversation_id: currentConversationId - } - - this.messages = [...this.messages, userMessage] - const messageContent = this.inputMessage.trim() - this.inputMessage = '' - this.isLoading = true - this.isWaitingForResponse = true - - try { - await tick() - this.scrollToUserMessage(userMessage.id) - - if (this.#useStreaming && this.#path) { - await this.handleStreamingMessage( - messageContent, - currentConversationId, - isNewConversation, - additionalInputs - ) - } else { - await this.handlePollingMessage( - messageContent, - currentConversationId, - isNewConversation, - additionalInputs - ) - } - } catch (error) { - console.error('Error running flow:', error) - sendUserToast('Failed to run flow: ' + error, true) - } finally { - if (!this.#useStreaming) { - this.isLoading = false - } - } - - await tick() - this.focusInput() - } - - private async handleStreamingMessage( - messageContent: string, - currentConversationId: string, - isNewConversation: boolean, - additionalInputs?: Record - ) { - // Close any existing EventSource - if (this.currentEventSource) { - this.currentEventSource.close() - } - - try { - const jobId = await this.#onRunFlow?.(messageContent, currentConversationId, additionalInputs) - if (!jobId) { - console.error('No jobId returned from onRunFlow') - return - } - this.currentJobId = jobId - - this.startPolling(currentConversationId, isNewConversation) - - this.#followJob(jobId, currentConversationId, { - accumulatedContent: '', - assistantMessageId: '', - streamOffset: undefined, - streamJobId: undefined - }) - } catch (error) { - console.error('Stream connection error:', error) - sendUserToast('Failed to connect to stream', true) - this.cleanup() - } - } - - // Opens an SSE connection on an already-running job. The server closes every - // stream after TIMEOUT_SSE_STREAM, so a timeout re-enters here with the same - // job and turn state rather than starting a new run. - #followJob(jobId: string, currentConversationId: string, turn: StreamTurnState) { - const streamUrl = `/api/w/${this.#workspace()}/jobs_u/getupdate_sse/${jobId}` - const url = new URL(streamUrl, window.location.origin) - url.searchParams.set('poll_delay_ms', '50') - url.searchParams.set('fast', 'true') - url.searchParams.set('only_result', 'true') - if (turn.streamOffset !== undefined) { - url.searchParams.set('stream_offset', turn.streamOffset.toString()) - } - const eventSource = new EventSource(url.toString()) - this.currentEventSource = eventSource - let isCompleted = false - - eventSource.onmessage = async (event) => { - try { - const data = JSON.parse(event.data) - const type = data.type - - if (type === 'timeout') { - eventSource.close() - this.currentEventSource = undefined - this.#followJob(jobId, currentConversationId, turn) - return - } - - // Handle ping - just ignore - if (type === 'ping') { - return - } - - // Handle error - if (type === 'error') { - eventSource.close() - this.currentEventSource = undefined - console.error('SSE error:', data) - sendUserToast('Stream error: ' + (data.error || 'Unknown error'), true) - this.cleanup() - return - } - - // Handle not found - if (type === 'not_found') { - eventSource.close() - this.currentEventSource = undefined - console.error('Job not found') - sendUserToast('Job not found', true) - this.cleanup() - return - } - - if (type === 'update') { - if (data.flow_stream_job_id) { - this.currentJobId = data.flow_stream_job_id - if (data.flow_stream_job_id !== turn.streamJobId) { - const offsetFromOtherJob = - turn.streamJobId !== undefined && turn.streamOffset !== undefined - turn.streamJobId = data.flow_stream_job_id - if (offsetFromOtherJob) { - // The offset indexes the previous sub-job's stream (a retried last step - // gets a new one), so this connection skipped the new job's first chunks. - // Drop this delta and re-attach from the start of the new sub-job. - turn.streamOffset = undefined - eventSource.close() - this.currentEventSource = undefined - this.#followJob(jobId, currentConversationId, turn) - return - } - } - } - if (data.stream_offset !== undefined) { - turn.streamOffset = data.stream_offset - } - // Process new stream content - if (data.new_result_stream) { - // Stop polling since we are receiving last step streaming - this.stopPolling() - const { type, content: newContent, success } = parseStreamDeltas(data.new_result_stream) - turn.accumulatedContent += newContent - - // Create tool message if type is tool_result - if (type === 'tool_result') { - // set last message streaming to false - this.messages = this.messages.map((msg) => - msg.id === this.messages[this.messages.length - 1].id - ? { ...msg, streaming: false } - : msg - ) - - this.messages = [ - ...this.messages, - { - id: 'temp-' + randomUUID(), - content: newContent, - created_at: new Date().toISOString(), - created_seq: 0, - message_type: 'tool', - conversation_id: currentConversationId, - job_id: '', - loading: false, - streaming: false, - success - } - ] - // Reset assistant message ID since we are creating a tool message - turn.assistantMessageId = '' - turn.accumulatedContent = '' - } - - // Create message on first content - else if ( - type === 'message' && - turn.assistantMessageId.length === 0 && - turn.accumulatedContent.length > 0 - ) { - turn.assistantMessageId = 'temp-' + randomUUID() - this.messages = [ - ...this.messages, - { - id: turn.assistantMessageId, - content: turn.accumulatedContent, - created_at: new Date().toISOString(), - created_seq: 0, - message_type: 'assistant', - conversation_id: currentConversationId, - job_id: '', - loading: false, - streaming: true - } - ] - } else { - // Update existing message - this.messages = this.messages.map((msg) => - msg.id === turn.assistantMessageId - ? { ...msg, content: turn.accumulatedContent } - : msg - ) - } - } - - // Handle completion - if (data.completed) { - isCompleted = true - // Do a final poll to get all messages from database - if (this.selectedConversationId) { - await this.pollConversationMessages(this.selectedConversationId, { - removeTempMessages: true - }) - } - this.cleanup() - } - } - } catch (error) { - console.error('Error processing stream event:', error) - } - } - - eventSource.onerror = (error) => { - if (isCompleted) return - console.error('EventSource error:', error) - sendUserToast('Stream error occurred', true) - this.cleanup() - } - } - - private async handlePollingMessage( - messageContent: string, - currentConversationId: string, - isNewConversation: boolean, - additionalInputs?: Record - ) { - const jobId = await this.#onRunFlow?.(messageContent, currentConversationId, additionalInputs) - if (!jobId) { - console.error('No jobId returned from onRunFlow') - return - } - - // Store the current job ID so it can be cancelled - this.currentJobId = jobId - - if (isNewConversation) { - await this.refreshConversations() - } - - // Start polling for intermediate messages in non-streaming mode too - this.startPolling(currentConversationId) - this.pollJobResult(jobId) - } -} - -export const createFlowChatManager = () => new FlowChatManager() diff --git a/frontend/src/lib/components/flows/conversations/FlowConversationsSidebar.svelte b/frontend/src/lib/components/flows/conversations/FlowConversationsSidebar.svelte index 2f09bcebe1..02c2581db1 100644 --- a/frontend/src/lib/components/flows/conversations/FlowConversationsSidebar.svelte +++ b/frontend/src/lib/components/flows/conversations/FlowConversationsSidebar.svelte @@ -1,26 +1,72 @@
    @@ -31,11 +77,11 @@ unifiedSize="md" variant="subtle" startIcon={{ - icon: manager.isSidebarExpanded ? PanelLeftClose : PanelLeftOpen, + icon: expanded ? PanelLeftClose : PanelLeftOpen, classes: 'ml-[2px]' }} - onClick={() => (manager.isSidebarExpanded = !manager.isSidebarExpanded)} - iconOnly={!manager.isSidebarExpanded} + onClick={() => (expanded = !expanded)} + iconOnly={!expanded} btnClasses={'justify-start transition-all duration-150'} title="Conversations" > @@ -45,9 +91,9 @@ unifiedSize="md" variant="subtle" startIcon={{ icon: Plus, classes: 'ml-[2px]' }} - onClick={() => manager.createConversation({ clearMessages: true })} + onClick={newChat} title="Start new conversation" - iconOnly={!manager.isSidebarExpanded} + iconOnly={!expanded} btnClasses={'justify-start transition-all duration-150 whitespace-nowrap'} >
    New chat
    @@ -56,50 +102,69 @@
    - {#if !manager.isSidebarExpanded} + {#if !expanded}
    {/if} -
    +
    + {#if draftShown && expanded} +
    + +
    + {/if} - {#snippet customRow({ item: conversation, hover })} - {#if manager.isSidebarExpanded} + {#snippet customRow({ item: conversation })} + {#if expanded}
    diff --git a/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte b/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte index 441ed52373..6a6e731c11 100644 --- a/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte @@ -84,7 +84,6 @@ onEditInForkClick } from '$lib/utils/editInFork' import { isCloudHosted } from '$lib/cloud' - import { agentStreamingEnabled } from '$lib/components/flows/agentFormFields' let flow: Flow | undefined = $state() let can_write = $state(false) @@ -523,12 +522,6 @@ let showEditButtons = $state(false) let mainButtons = $derived(getMainButtons(flow, args)) let chatInputEnabled = $derived(flow?.value?.chat_input_enabled ?? false) - let shouldUseStreaming = $derived.by(() => { - const modules = flow?.value?.modules - const lastModule = modules && modules.length > 0 ? modules[modules.length - 1] : undefined - if (lastModule?.value?.type !== 'aiagent') return false - return agentStreamingEnabled(lastModule.value) - }) @@ -701,7 +694,6 @@ onRunFlow={runFlowForChat} {deploymentInProgress} path={flow?.path ?? ''} - useStreaming={shouldUseStreaming} inputSchema={flow?.schema} /> {:else} diff --git a/frontend/svelte.config.js b/frontend/svelte.config.js index 1752fed97c..15634551e6 100644 --- a/frontend/svelte.config.js +++ b/frontend/svelte.config.js @@ -40,7 +40,10 @@ const config = { }, alias: { $system_prompts: '../system_prompts/auto-generated', - $oauth_connect_registry: '../backend/oauth_connect.json' + $oauth_connect_registry: '../backend/oauth_connect.json', + // The flow chat runs on the published SDK's source, so the product and the + // package share one implementation (vite.config.js allows serving it). + 'windmill-chat': '../chat-sdk/src/index.ts' } }, diff --git a/frontend/vite.config.js b/frontend/vite.config.js index a95ada7880..0140c38739 100644 --- a/frontend/vite.config.js +++ b/frontend/vite.config.js @@ -1,6 +1,7 @@ import { sveltekit } from '@sveltejs/kit/vite' import { existsSync, readFileSync } from 'fs' import { fileURLToPath } from 'url' +import { searchForWorkspaceRoot } from 'vite' import mkcert from 'vite-plugin-mkcert' const file = fileURLToPath(new URL('package.json', import.meta.url)) @@ -205,6 +206,13 @@ const config = { ], port: parseInt(process.env.FRONTEND_PORT) || 3000, cors: { origin: '*' }, + // `windmill-chat` (svelte.config.js alias) lives outside the frontend root. + fs: { + allow: [ + searchForWorkspaceRoot(process.cwd()), + fileURLToPath(new URL('../chat-sdk', import.meta.url)) + ] + }, proxy: { '^/\\.well-known/.*': { target: remoteUrl, From 129c04559548cd1bcf67758ec416fb2a48e7b928 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 15 Sep 2026 16:33:47 +0200 Subject: [PATCH 26/44] fix(cli): keep the workspace color when settings are synced from git (#11144) * fix(cli): keep the workspace color when settings are synced from git Co-Authored-By: Claude Fable 5.1 * docs(cli): name the sync direction consistently in the identity-field comments Co-Authored-By: Claude Fable 5.1 * fix(cli): apply the workspace color from settings.yaml only when the file sets one Co-Authored-By: Claude Fable 5.1 --------- Co-authored-by: Claude Fable 5.1 --- cli/src/commands/sync/sync.ts | 21 +++++++ cli/src/core/settings.ts | 19 +++++-- cli/test/push_diff_convergence_unit.test.ts | 26 ++++++++- ..._workspace_settings_identity_unit.test.ts} | 55 +++++++++++++++++-- 4 files changed, 108 insertions(+), 13 deletions(-) rename cli/test/{push_workspace_settings_name_unit.test.ts => push_workspace_settings_identity_unit.test.ts} (54%) diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index 287dd6ae12..38b9e87f84 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -2542,6 +2542,21 @@ export function preservePendingScriptLocks( } } +// `sync push` never applies the workspace's display name from settings.yaml and +// applies its color only when the local file carries one (see +// pushWorkspaceSettings), so on a push the fields it would not apply must +// compare equal, or the row is listed on every run. +const isWorkspaceSettingsFile = (p: string) => + /^settings(\.[^./\\]+)?\.(yaml|json)$/.test(p); +function stripUnappliedSettingsFields(local: any, remote: any) { + delete local?.name; + delete remote?.name; + if (local?.color == null) { + delete local?.color; + delete remote?.color; + } +} + export async function compareDynFSElement( els1: DynFSElement, els2: DynFSElement | undefined, @@ -2757,6 +2772,9 @@ export async function compareDynFSElement( delete parsedV?.enabled; delete parsedM2?.enabled; } + if (isEls1Remote === false && isWorkspaceSettingsFile(k)) { + stripUnappliedSettingsFields(parsedV, parsedM2); + } if (deepEqual(parsedV, parsedM2)) { continue; } @@ -2771,6 +2789,9 @@ export async function compareDynFSElement( delete before?.enabled; delete after?.enabled; } + if (isEls1Remote === false && isWorkspaceSettingsFile(k)) { + stripUnappliedSettingsFields(after, before); + } if (deepEqual(before, after)) { continue; } diff --git a/cli/src/core/settings.ts b/cli/src/core/settings.ts index 66fc77d230..4c95090bcf 100644 --- a/cli/src/core/settings.ts +++ b/cli/src/core/settings.ts @@ -214,9 +214,15 @@ export async function pushWorkspaceSettings( } // Exclude fields that are never applied here: slack_team_id/slack_name are OAuth-only, - // and name is not applied on pull (see below), so a name-only diff stays a no-op. + // and name is never applied (see below), so a name-only diff stays a no-op. color is + // applied only when the file carries it, so an unset one leaves the comparison too. const { slack_team_id: _lst, slack_name: _lsn, name: _ln, ...comparableLocal } = localSettings; const { slack_team_id: _rst, slack_name: _rsn, name: _rn, ...comparableRemote } = settings; + const colorManaged = localSettings.color != null; + if (!colorManaged) { + delete comparableLocal.color; + delete comparableRemote.color; + } if (isSuperset(comparableLocal, comparableRemote)) { log.debug(`Workspace settings are up to date`); return; @@ -351,10 +357,9 @@ export async function pushWorkspaceSettings( }); } - // Workspace display name is intentionally not applied on pull: settings.yaml is shared - // across a repo's branches, so applying it would let one workspace's name overwrite - // another's when both sync the same repo. It stays in the file (written on push), but a - // live workspace is only renamed by its owner. + // Workspace display name is intentionally never applied by `sync push`: settings.yaml is + // shared across a repo's branches, so applying it would let one workspace's name overwrite + // another's when both sync the same repo. `sync pull` still records it. if (localSettings.mute_critical_alerts != settings.mute_critical_alerts) { log.debug(`Updating mute critical alerts...`); @@ -366,7 +371,9 @@ export async function pushWorkspaceSettings( }); } - if (localSettings.color != settings.color) { + // A color is applied only when the file carries one: `sync pull` omits the key for a + // workspace without a color, so an unset key means "not managed by git", never "clear". + if (colorManaged && localSettings.color != settings.color) { log.debug(`Updating workspace color...`); await wmill.changeWorkspaceColor({ workspace, diff --git a/cli/test/push_diff_convergence_unit.test.ts b/cli/test/push_diff_convergence_unit.test.ts index 14e248c03e..67f6d5b4dd 100644 --- a/cli/test/push_diff_convergence_unit.test.ts +++ b/cli/test/push_diff_convergence_unit.test.ts @@ -64,6 +64,7 @@ async function diff( remoteEl: Mock, skips: Record, parentOwnsScheduleEnabled?: (scheduleFilePath: string) => boolean, + isEls1Remote = false, ) { const { changes } = await compareDynFSElement( localEl as any, @@ -76,7 +77,7 @@ async function diff( false, undefined, undefined, - false, + isEls1Remote, false, parentOwnsScheduleEnabled, ); @@ -296,3 +297,26 @@ test("push: checkout inline names stay inside the flow folder", async () => { await checkoutInlineNames(join(process.cwd(), "missing.yaml")), ).toEqual({}); }); + +// A push never applies the workspace's display name and applies its color only +// when the local file carries one (see pushWorkspaceSettings), so a file that +// differs only in what would not be applied is not a push change; a pull still +// rewrites the file. +test("push: settings.yaml differing only by name or an unset color is not a change", async () => { + const remote = local({ + "settings.yaml": "name: prod\ncolor: '#ff0000'\nerror_handler: null\n", + }); + const unsetColor = local({ + "settings.yaml": "name: staging\nerror_handler: null\n", + }); + const skips = { includeSettings: true }; + expect(await diff(unsetColor, remote, skips)).toEqual([]); + expect(await diff(remote, unsetColor, skips, undefined, true)).toEqual([ + "edited settings.yaml", + ]); + + const otherColor = local({ + "settings.yaml": "name: staging\ncolor: '#00ff00'\nerror_handler: null\n", + }); + expect(await diff(otherColor, remote, skips)).toEqual(["edited settings.yaml"]); +}); diff --git a/cli/test/push_workspace_settings_name_unit.test.ts b/cli/test/push_workspace_settings_identity_unit.test.ts similarity index 54% rename from cli/test/push_workspace_settings_name_unit.test.ts rename to cli/test/push_workspace_settings_identity_unit.test.ts index ef76ab7aec..c583a53282 100644 --- a/cli/test/push_workspace_settings_name_unit.test.ts +++ b/cli/test/push_workspace_settings_identity_unit.test.ts @@ -1,23 +1,32 @@ /** - * Regression guard: a pull (pushWorkspaceSettings) must not apply the workspace - * display name from settings.yaml. Rationale lives at the apply site in settings.ts. + * Regression guard: `sync push` (pushWorkspaceSettings) must never apply the + * workspace display name from settings.yaml, and must apply the color only when + * the file carries one. Rationale lives at the apply sites in settings.ts. */ import { expect, test, describe, beforeEach, mock } from "bun:test"; let changeWorkspaceNameCalls: unknown[] = []; +let changeWorkspaceColorCalls: unknown[] = []; let editWebhookCalls: unknown[] = []; let remoteName = ""; +let remoteColor: string | undefined = undefined; let remoteWebhook: string | undefined = undefined; // Every wmill.* call reachable from pushWorkspaceSettings is stubbed so the -// function runs without a backend; only the two we assert on record calls. +// function runs without a backend; only the three we assert on record calls. mock.module("../gen/services.gen.ts", () => ({ - getSettings: async (_a: { workspace: string }) => ({ webhook: remoteWebhook }), + getSettings: async (_a: { workspace: string }) => ({ + webhook: remoteWebhook, + color: remoteColor, + }), getWorkspaceName: async (_a: { workspace: string }) => remoteName, changeWorkspaceName: async (a: unknown) => { changeWorkspaceNameCalls.push(a); }, + changeWorkspaceColor: async (a: unknown) => { + changeWorkspaceColorCalls.push(a); + }, editWebhook: async (a: unknown) => { editWebhookCalls.push(a); }, @@ -30,7 +39,6 @@ mock.module("../gen/services.gen.ts", () => ({ editWorkspaceDefaultApp: async () => {}, editDefaultScripts: async () => {}, workspaceMuteCriticalAlertsUi: async () => {}, - changeWorkspaceColor: async () => {}, updateOperatorSettings: async () => {}, editDataTableConfig: async () => {}, editSlackCommand: async () => {}, @@ -40,13 +48,15 @@ mock.module("../gen/services.gen.ts", () => ({ const { pushWorkspaceSettings } = await import("../src/core/settings.ts"); -describe("pushWorkspaceSettings workspace name", () => { +describe("pushWorkspaceSettings workspace identity", () => { const ws = "phoenix"; beforeEach(() => { changeWorkspaceNameCalls = []; + changeWorkspaceColorCalls = []; editWebhookCalls = []; remoteName = "phoenix"; + remoteColor = undefined; remoteWebhook = undefined; }); @@ -69,4 +79,37 @@ describe("pushWorkspaceSettings workspace name", () => { expect(editWebhookCalls.length).toBe(0); expect(changeWorkspaceNameCalls.length).toBe(0); }); + + test("a settings.yaml without a color key does not clear the workspace color", async () => { + remoteColor = "#ff0000"; + remoteWebhook = "https://old"; + await pushWorkspaceSettings(ws, "settings", undefined, { + name: "phoenix", + webhook: "https://new", + }); + expect(editWebhookCalls.length).toBe(1); + expect(changeWorkspaceColorCalls.length).toBe(0); + }); + + test("a color in settings.yaml is applied when it differs from the workspace", async () => { + remoteColor = "#ff0000"; + await pushWorkspaceSettings(ws, "settings", undefined, { + name: "phoenix", + color: "#00ff00", + }); + expect(editWebhookCalls.length).toBe(0); + expect(changeWorkspaceColorCalls).toEqual([ + { workspace: ws, requestBody: { color: "#00ff00" } }, + ]); + }); + + test("a color matching the workspace is a complete no-op", async () => { + remoteColor = "#ff0000"; + await pushWorkspaceSettings(ws, "settings", undefined, { + name: "phoenix", + color: "#ff0000", + }); + expect(editWebhookCalls.length).toBe(0); + expect(changeWorkspaceColorCalls.length).toBe(0); + }); }); From 781b5a57e81eb721d97d7b87e23dd84f23895400 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 15 Sep 2026 16:36:16 +0200 Subject: [PATCH 27/44] fix(apps): run-mode inline app component uses only pinned content (#11135) Co-authored-by: Claude Opus 4.8 --- backend/tests/app_run_mode_lock_strip.rs | 240 +++++++++++++++++++++++ backend/windmill-api/src/apps.rs | 52 +++++ 2 files changed, 292 insertions(+) create mode 100644 backend/tests/app_run_mode_lock_strip.rs diff --git a/backend/tests/app_run_mode_lock_strip.rs b/backend/tests/app_run_mode_lock_strip.rs new file mode 100644 index 0000000000..e2bbf631db --- /dev/null +++ b/backend/tests/app_run_mode_lock_strip.rs @@ -0,0 +1,240 @@ +//! Regression: run mode of `execute_component`'s no-id inline-`raw_code` arm runs +//! *only* the `rawscript/`-pinned `content`, dropping the caller `hash`, +//! `lock`, `modules` and `dedicated_worker` and deriving `path` server-side — +//! all of which would otherwise run or install unpinned code as the app identity. +//! Preview mode keeps honoring the caller's fields. + +use serde_json::json; +use sha2::{Digest, Sha256}; +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(b: reqwest::RequestBuilder, token: &str) -> reqwest::RequestBuilder { + b.header("Authorization", format!("Bearer {}", token)) +} + +const CONTENT: &str = "print('benign')\n"; +// A caller lock whose presence is the whole point: if it reaches the job, the +// worker installs it. The value only needs to be recognizable in `v2_job`. +const CALLER_LOCK: &str = "evilpkg @ file:///tmp/attacker-controlled-sdist"; +// A non-codebase-sentinel hash: if it reaches the job as `runnable_id`, the +// worker fetches (and runs) a deployed script by hash instead of the pinned +// content. It need not resolve to a real row — the guard is that it never +// becomes `runnable_id`. +const CALLER_HASH: i64 = 123456789; +// A caller path in someone else's namespace: if it reaches the job as +// `runnable_path` it redirects where the pinned content's relative imports +// resolve. Run mode must instead derive the path from `/`. +const CALLER_PATH: &str = "u/attacker/evil/comp"; + +/// The pin key `execute_component` computes for a no-id inline script: +/// `rawscript/`. +fn rawscript_pin(content: &str) -> String { + let mut h = Sha256::new(); + h.update(content); + format!("rawscript/{:x}", h.finalize()) +} + +fn inline_raw_code(hash: Option, dedicated: bool) -> serde_json::Value { + let mut rc = json!({ + "language": "python3", + "content": CONTENT, + "path": CALLER_PATH, + "lock": CALLER_LOCK, + "modules": { + "m.py": { "content": "print('x')\n", "language": "python3", "lock": CALLER_LOCK } + } + }); + if let Some(h) = hash { + rc["hash"] = json!(h); + } + if dedicated { + rc["dedicated_worker"] = json!(true); + } + rc +} + +/// Fetch `(raw_lock, args-has-_MODULES, runnable_id, tag, runnable_path)` for an +/// enqueued job. +async fn job_fields( + db: &Pool, + uuid: uuid::Uuid, +) -> anyhow::Result<(Option, bool, Option, String, Option)> { + Ok(sqlx::query_as( + "SELECT raw_lock, (args ? '_MODULES'), runnable_id, tag, runnable_path \ + FROM v2_job WHERE id = $1", + ) + .bind(uuid) + .fetch_one(db) + .await?) +} + +#[sqlx::test(fixtures("base"))] +async fn test_run_mode_strips_caller_lock_and_modules(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = format!("http://localhost:{port}/api/w/test-workspace"); + + let app_path = "u/test-user/lockstrip"; + let pin = format!("comp:{}", rawscript_pin(CONTENT)); + + // Deployed Viewer-mode app whose only runnable is an inline script pinned by + // content hash and with no `app_script` row — the legacy `rawscript/` + // case that reaches the no-id run-mode arm this fix touches. + let resp = authed(client().post(format!("{ws}/apps/create")), "SECRET_TOKEN") + .json(&json!({ + "path": app_path, + "summary": "", + "value": {}, + "policy": { + "execution_mode": "viewer", + "triggerables_v2": { pin: { "static_inputs": {}, "one_of_inputs": {} } } + } + })) + .send() + .await?; + assert_eq!(resp.status(), 201, "create app: {}", resp.text().await?); + + // Run mode (no `force_viewer_static_fields`): the pin authorizes the run, but + // every caller field that selects, installs, or routes code — hash, lock, + // modules, dedicated_worker — must be dropped. + let resp = authed( + client().post(format!("{ws}/apps_u/execute_component/{app_path}")), + "SECRET_TOKEN_2", + ) + .json(&json!({ + // The args map is the other injection channel: an inline run is a + // `JobKind::Preview` job, so the worker/executors read `_MODULES` and + // `_TEMP_SCRIPT_REFS` back out of the job args. Both must be stripped. + "args": { + "_MODULES": { "m.py": { "content": "print('evil')\n", "language": "python3" } }, + "_TEMP_SCRIPT_REFS": { "../evil": "deadbeef" } + }, + "component": "comp", + "raw_code": inline_raw_code(Some(CALLER_HASH), true) + })) + .send() + .await?; + let status = resp.status(); + let body = resp.text().await?; + assert_eq!( + status, 200, + "run-mode pinned inline run must be accepted: {body}" + ); + let uuid = uuid::Uuid::parse_str(body.trim())?; + let (raw_lock, has_modules, runnable_id, tag, runnable_path) = job_fields(&db, uuid).await?; + assert_eq!( + raw_lock, None, + "run mode must strip the caller-supplied lock" + ); + assert!( + !has_modules, + "run mode must strip caller modules (both `raw_code.modules` and an `_MODULES` arg)" + ); + let has_temp_refs: bool = + sqlx::query_scalar("SELECT (args ? '_TEMP_SCRIPT_REFS') FROM v2_job WHERE id = $1") + .bind(uuid) + .fetch_one(&db) + .await?; + assert!( + !has_temp_refs, + "run mode must strip a caller `_TEMP_SCRIPT_REFS` arg (relative-import redirect)" + ); + assert_eq!( + runnable_id, None, + "run mode must strip the caller-supplied hash (no substituting a deployed script by hash)" + ); + assert!( + !tag.starts_with("dedi:"), + "run mode must strip caller `dedicated_worker` (no routing to a path-keyed dedicated worker), got tag {tag:?}" + ); + assert_eq!( + runnable_path.as_deref(), + Some(format!("{app_path}/comp").as_str()), + "run mode must derive the path server-side, not trust the caller's (relative-import base)" + ); + + // Preview mode (editor): the caller runs their own code as themselves, so the + // lock and modules are honored — the `/jobs/run/preview`-equivalent path. + let resp = authed( + client().post(format!("{ws}/apps_u/execute_component/{app_path}")), + "SECRET_TOKEN_2", + ) + .json(&json!({ + "args": {}, + "component": "comp", + "raw_code": inline_raw_code(None, false), + "force_viewer_static_fields": {} + })) + .send() + .await?; + let status = resp.status(); + let body = resp.text().await?; + assert_eq!(status, 200, "preview must be accepted: {body}"); + let uuid = uuid::Uuid::parse_str(body.trim())?; + let (raw_lock, has_modules, _, _, _) = job_fields(&db, uuid).await?; + assert_eq!( + raw_lock.as_deref(), + Some(CALLER_LOCK), + "preview must keep the caller-supplied lock" + ); + assert!(has_modules, "preview must keep the caller-supplied modules"); + + Ok(()) +} + +/// A bare `rawscript/` policy key (no `:` prefix, as `empty_triggerables` +/// migrates v1 policies) matches for any `component`, so run mode must not let a +/// path-traversing `component` steer the server-derived `runnable_path`. +#[sqlx::test(fixtures("base"))] +async fn test_run_mode_rejects_traversal_component(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = format!("http://localhost:{port}/api/w/test-workspace"); + + let app_path = "u/test-user/lockstrip_bare"; + // Bare key: no `comp:` prefix, so the pin matches regardless of `component`. + let resp = authed(client().post(format!("{ws}/apps/create")), "SECRET_TOKEN") + .json(&json!({ + "path": app_path, + "summary": "", + "value": {}, + "policy": { + "execution_mode": "viewer", + "triggerables_v2": { rawscript_pin(CONTENT): { "static_inputs": {}, "one_of_inputs": {} } } + } + })) + .send() + .await?; + assert_eq!(resp.status(), 201, "create app: {}", resp.text().await?); + + // A component that isn't a single plain segment steers the derived path's + // base: separators and `..` traverse, and an empty one shifts it up a level. + for bad in ["../../u/attacker/evil", "..", "a/b", ""] { + let resp = authed( + client().post(format!("{ws}/apps_u/execute_component/{app_path}")), + "SECRET_TOKEN_2", + ) + .json(&json!({ + "args": {}, + "component": bad, + "raw_code": inline_raw_code(None, false) + })) + .send() + .await?; + let status = resp.status(); + let body = resp.text().await?; + assert_eq!( + status, 400, + "run mode must reject component {bad:?}: got {status}: {body}" + ); + } + + Ok(()) +} diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index cc4c5079f9..6af6bbc37c 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -3853,6 +3853,26 @@ fn digest(code: &str) -> String { format!("rawscript/{:x}", result) } +/// Canonical `runnable_path` for a run-mode no-id inline app component: +/// `/` — byte-for-byte what the runtime frontend sends. It is +/// the relative-import base, so the caller must not steer it: reject a `component` +/// that isn't a single non-empty path segment (empty, a separator, or `.`/`..` +/// would walk the base out of the app, and a bare `rawscript/` policy key +/// does not pin the component). +fn inline_run_path(app_path: &str, component: &str) -> Result { + if component.is_empty() + || component.contains('/') + || component.contains('\\') + || component == "." + || component == ".." + { + return Err(Error::BadRequest( + "component id must be a single non-empty path segment".to_string(), + )); + } + Ok(format!("{app_path}/{component}")) +} + async fn get_on_behalf_details_from_policy_and_authed( policy: &Policy, opt_authed: &Option, @@ -4257,6 +4277,14 @@ async fn execute_component( let resolved_delete_secs = resolve_delete_after_secs(None, policy_triggerables.delete_after_secs); + // `_MODULES` and `_TEMP_SCRIPT_REFS` are server-injected control keys (into + // `extra`) that the worker reads back for a `Preview` job — which an inline run + // is. A caller supplying them in `args` would inject module content/locks or + // redirect relative-import resolution, unpinned, as the app identity. Drop them; + // legitimate values ride in `extra`, never the request `args`. + payload.args.remove("_MODULES"); + payload.args.remove("_TEMP_SCRIPT_REFS"); + let (mut args, job_id) = build_args( policy, policy_triggerables, @@ -4294,6 +4322,7 @@ async fn execute_component( } .filter(|t| !t.is_empty()) }; + let component = payload.component.clone(); let (job_payload, tag, _runnable_on_behalf_of) = match (payload.path, payload.raw_code, payload.id) { // flow or script: @@ -4304,6 +4333,29 @@ async fn execute_component( // `app_script` table (legacy `rawscript/`-keyed triggerables). (None, Some(raw_code), None) => { let tag = resolved_inline_tag(raw_code.tag.clone()); + let raw_code = if is_preview { + // Preview (editor / `wmill app dev`): the caller runs their own + // code, like `/jobs/run/preview` — honored verbatim. + raw_code + } else { + // Run mode. Legacy back-compat only: current deploys assign an + // `app_script` id (reduce_app) and take the `Some(id)` arm; + // drop this branch once id-less deployed apps are gone. + // + // Only `content` is pinned (`rawscript/`), so keep just + // that plus `language`/`cache_ttl`, derive `path` server-side + // (`inline_run_path`), and default the rest: a caller `hash`/ + // `lock`/`modules`/`path`/`dedicated_worker` would otherwise run + // or install unpinned code as the app identity. Reconstructing + // (vs nulling) keeps a new field defaulting safe. + RawCode { + content: raw_code.content, + language: raw_code.language, + path: Some(inline_run_path(path, &component)?), + cache_ttl: raw_code.cache_ttl, + ..Default::default() + } + }; (JobPayload::Code(raw_code), tag, None) } // inline script: run mode (deployed app) with an entry in `app_script`. From 796b6e5297d8cceb842ec097f33ec1c3115058bd Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 15 Sep 2026 17:48:33 +0200 Subject: [PATCH 28/44] feat: back AI sessions up to the workspace object storage (#11116) * feat: back AI sessions up to the workspace object storage Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01SG5qEPM6Fmf7VerXS5nnWp * fix: bind the backup key to the user and pack pushes within the server caps Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01SG5qEPM6Fmf7VerXS5nnWp * fix: keep refused and unavailable marks, one mark per key, stream the flush Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01SG5qEPM6Fmf7VerXS5nnWp * fix: settle only fully sent sessions, keep removals while backups are off, cap pull bodies Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01SG5qEPM6Fmf7VerXS5nnWp * fix: bound removal marks while backups are off and stale the sync rows instead of dropping them Co-Authored-By: Claude Fable 5.1 * fix: retry a lost lock, cap nested push lists and oversized pieces, drop a stale copy of a chat that outgrew the backup Co-Authored-By: Claude Fable 5.1 * fix: cap pieces per push, size requests in UTF-8, keep a move's removal for an off workspace, disclose the restore counter Co-Authored-By: Claude Fable 5.1 * fix: file a move's removal only once the new copy landed, retire marks through the sync row Co-Authored-By: Claude Fable 5.1 * fix: keep a session marked while deletes are carried over, drop only gone sessions' marks Co-Authored-By: Claude Fable 5.1 * fix: record what a refused flush already stored, stop early when every mark is retired Co-Authored-By: Claude Fable 5.1 * fix: restore past another workspace's removal mark, file a move's removal before its row, bound the first pulled session Co-Authored-By: Claude Fable 5.1 * fix: re-key the backups on workspace key rotation, accept only base64 images, carry a delete on the sync row when its mark cannot be written Co-Authored-By: Claude Fable 5.1 * fix: durable conditional re-key of session backups, re-push on a storage switch Co-Authored-By: Claude Fable 5.1 * fix: fail a push the key rotated under, settle no session split across storages, narrow the re-key module Co-Authored-By: Claude Fable 5.1 * fix: keep a delete filed during a push, bound the pull and re-key listings Co-Authored-By: Claude Fable 5.1 * fix: bound the session listing, mark the store's own user on a write that lands after a user switch Co-Authored-By: Claude Fable 5.1 * fix: list sessions through per-session index markers, hold a session's parts back after a failed one Co-Authored-By: Claude Fable 5.1 * fix: record a rotation on every build, list a session only on the part that completes its push Co-Authored-By: Claude Fable 5.1 * fix: leave an object larger than any push writes unread Co-Authored-By: Claude Fable 5.1 * fix: read each object against its listed size, carry a dirty mark that cannot be written on the sync row Co-Authored-By: Claude Fable 5.1 * fix: note the storages the re-key walk completed on, reach another user's rows on a failed mark, read a head at its cap Co-Authored-By: Claude Fable 5.1 * fix: read the replaced key under its row lock, carry a refused dirty bump on the sync row Co-Authored-By: Claude Fable 5.1 * fix: pull a session that outgrew one answer in pages, imported only whole Co-Authored-By: Claude Fable 5.1 * fix: build a pull page from the smallest keys of the whole listing, stage each page as it lands Co-Authored-By: Claude Fable 5.1 * fix: re-record a key rotated back to, admit earlier-page images, restage over a cut-short restore Co-Authored-By: Claude Fable 5.1 * refactor: delete the backups on key rotation instead of re-keying them Co-Authored-By: Claude Fable 5.1 * fix: end a pull page before an object that grew since the listing, prune what a cut-short restore staged Co-Authored-By: Claude Fable 5.1 * fix: delete the backups before the key commits, skip a planted object whatever its listing says, lock a restore across tabs, prune stale artifact versions Co-Authored-By: Claude Fable 5.1 * fix: keep the backups under a prefix named by the key, delete the previous key's prefix after the commit Co-Authored-By: Claude Fable 5.1 * fix: name the backup prefix by a generation the rotation bumps, never write an older record over a newer one on restore Co-Authored-By: Claude Fable 5.1 * fix: retire a removal only against the storage holding the backup, restart a paged pull whose listing moved Co-Authored-By: Claude Fable 5.1 * fix: fingerprint a pull page before reading it, answer the backup generation apart from the storage identity Co-Authored-By: Claude Fable 5.1 * fix: answer needs_head for a headless session push, prune restaged pieces by id Co-Authored-By: Claude Fable 5.1 * fix: serialize a session's push and removal, open whole pushes with the head, prune only own restores Co-Authored-By: Claude Fable 5.1 * fix: require a head on a whole push, prune before the record lands, restore only under Web Locks Co-Authored-By: Claude Fable 5.1 * fix: incremental pushes ride on a listed session, removals wait for every storage holding a copy Co-Authored-By: Claude Fable 5.1 * fix: a whole push replaces the backup under a per-push token, a pull page is checked after its reads Co-Authored-By: Claude Fable 5.1 * fix: fingerprint a pull page by entity tag and version too Co-Authored-By: Claude Fable 5.1 * fix: a moved session's removal mark names the storages holding the old copy Co-Authored-By: Claude Fable 5.1 * fix: restore a workspace family together, the newest copy of a moved session winning Co-Authored-By: Claude Fable 5.1 * fix: name the marker by the session's move count, abort a family restore a listing failed in Co-Authored-By: Claude Fable 5.1 * fix: list the family again before a restored record lands, require the pull fingerprint Co-Authored-By: Claude Fable 5.1 * fix: list the whole family once per restored workspace, off members included Co-Authored-By: Claude Fable 5.1 * fix: a push split over parts, incremental too, unlists the session until its last part Co-Authored-By: Claude Fable 5.1 * fix: refuse a partial push part that names no push Co-Authored-By: Claude Fable 5.1 * fix: keep a refused bump for a session with no row yet, probe an off workspace again Co-Authored-By: Claude Fable 5.1 * fix: backfill row-carried bumps after a reload, ask an off workspace again on a timer Co-Authored-By: Claude Fable 5.1 * fix: backfill a row for its bumps only when it carries some Co-Authored-By: Claude Fable 5.1 * fix: a backfilled mark that cannot be written counts from the page's counter Co-Authored-By: Claude Fable 5.1 * docs: say an off workspace is asked again, in the mirror's comments Co-Authored-By: Claude Fable 5.1 * chore: update ee-repo-ref to 1c1dab33563c4907aff8b0da825fb66db60af82a This commit updates the EE repository reference after PR #796 was merged in windmill-ee-private. Previous ee-repo-ref: 289b477ca3fc993da06ec09b11c8f55d5e4e39c1 New ee-repo-ref: 1c1dab33563c4907aff8b0da825fb66db60af82a Automated by sync-ee-ref workflow. * fix: unlist a session while an incremental push changes more than one object Co-Authored-By: Claude Fable 5.1 --------- Co-authored-by: Claude Fable 5.1 Co-authored-by: windmill-internal-app[bot] --- backend/ee-repo-ref.txt | 2 +- ...518_ai_sessions_backup_generation.down.sql | 1 + ...91518_ai_sessions_backup_generation.up.sql | 4 + backend/summarized_schema.txt | 2 +- backend/tests/ai_sessions.rs | 1085 +++++++++++ .../src/ai_session_backups.rs | 150 ++ backend/windmill-api-workspaces/src/lib.rs | 4 +- .../windmill-api-workspaces/src/workspaces.rs | 32 + backend/windmill-api/openapi.yaml | 296 +++ backend/windmill-api/src/ai.rs | 7 + backend/windmill-api/src/ai_sessions.rs | 1316 +++++++++++++ backend/windmill-api/src/lib.rs | 2 + backend/windmill-api/src/workspaces.rs | 6 + backend/windmill-common/src/variables.rs | 7 +- backend/windmill-object-store/src/lib.rs | 4 +- docs/ai-session-backups.md | 266 +++ .../lib/components/InstanceSettings.svelte | 22 +- .../copilot/chat/HistoryManager.svelte.ts | 173 +- .../copilot/chat/artifacts/artifactsDB.ts | 105 +- .../sessions/sessionMirror.svelte.ts | 1644 +++++++++++++++++ .../components/sessions/sessionMirror.test.ts | 1449 +++++++++++++++ .../sessions/sessionMirrorPlan.test.ts | 204 ++ .../components/sessions/sessionMirrorPlan.ts | 309 ++++ .../sessions/sessionMirrorSignal.ts | 42 + .../sessions/sessionState.svelte.ts | 77 +- .../sessions/sessionStateIndexedDb.test.ts | 34 + .../workspaceSettings/AISettings.svelte | 33 +- frontend/src/lib/userScopedDb.ts | 20 +- frontend/src/lib/userScopedStorage.ts | 14 +- .../src/routes/(root)/(logged)/+layout.svelte | 11 + 30 files changed, 7280 insertions(+), 41 deletions(-) create mode 100644 backend/migrations/20260915091518_ai_sessions_backup_generation.down.sql create mode 100644 backend/migrations/20260915091518_ai_sessions_backup_generation.up.sql create mode 100644 backend/tests/ai_sessions.rs create mode 100644 backend/windmill-api-workspaces/src/ai_session_backups.rs create mode 100644 backend/windmill-api/src/ai_sessions.rs create mode 100644 docs/ai-session-backups.md create mode 100644 frontend/src/lib/components/sessions/sessionMirror.svelte.ts create mode 100644 frontend/src/lib/components/sessions/sessionMirror.test.ts create mode 100644 frontend/src/lib/components/sessions/sessionMirrorPlan.test.ts create mode 100644 frontend/src/lib/components/sessions/sessionMirrorPlan.ts create mode 100644 frontend/src/lib/components/sessions/sessionMirrorSignal.ts diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 3adfe42b2f..7acd297dac 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -ccada062c072d7b74894b63863728fd1ef9bdffd \ No newline at end of file +1c1dab33563c4907aff8b0da825fb66db60af82a diff --git a/backend/migrations/20260915091518_ai_sessions_backup_generation.down.sql b/backend/migrations/20260915091518_ai_sessions_backup_generation.down.sql new file mode 100644 index 0000000000..2662a9f709 --- /dev/null +++ b/backend/migrations/20260915091518_ai_sessions_backup_generation.down.sql @@ -0,0 +1 @@ +ALTER TABLE workspace_settings DROP COLUMN IF EXISTS ai_sessions_backup_generation; diff --git a/backend/migrations/20260915091518_ai_sessions_backup_generation.up.sql b/backend/migrations/20260915091518_ai_sessions_backup_generation.up.sql new file mode 100644 index 0000000000..a62caefa60 --- /dev/null +++ b/backend/migrations/20260915091518_ai_sessions_backup_generation.up.sql @@ -0,0 +1,4 @@ +-- Bumped by every workspace key rotation: the AI session backups in the workspace storage +-- live under a prefix named by it, so a rotation moves to a fresh prefix and the previous +-- ones can be deleted at leisure without ever touching live objects. +ALTER TABLE workspace_settings ADD COLUMN ai_sessions_backup_generation BIGINT NOT NULL DEFAULT 0; diff --git a/backend/summarized_schema.txt b/backend/summarized_schema.txt index 463afd0d33..b93a4bf6fc 100644 --- a/backend/summarized_schema.txt +++ b/backend/summarized_schema.txt @@ -234,7 +234,7 @@ workspace_protection_rule: workspace_id(char), name(char), rules(int), bypass_gr FK: (workspace_id) -> workspace(id) workspace_runnable_dependencies: flow_path(char), runnable_path(char), script_hash(bigint), runnable_is_flow(bool), workspace_id(char), app_path(char), id(bigint) FK: (app_path, workspace_id) -> app(path, workspace_id) | (flow_path, workspace_id) -> flow(path, workspace_id) -workspace_settings: workspace_id(char), slack_team_id(char), slack_name(char), slack_command_script(char), slack_email(char), customer_id(char), plan(char), webhook(text), ai_config(jsonb), large_file_storage(jsonb), git_sync(jsonb), default_app(char), default_scripts(jsonb), deploy_ui(jsonb), mute_critical_alerts(bool), color(char), operator_settings(jsonb), teams_command_script(text), teams_team_id(text), teams_team_name(text), git_app_installations(jsonb), ducklake(jsonb), slack_oauth_client_id(char), slack_oauth_client_secret(char), datatable(jsonb), teams_team_guid(text), auto_invite(jsonb), error_handler(jsonb), success_handler(jsonb), public_app_execution_limit_per_minute(int), dbt_warehouses(jsonb), guest_access_enabled(bool), guest_jwt_public_key(text), guest_jwt_jwks_url(text) +workspace_settings: workspace_id(char), slack_team_id(char), slack_name(char), slack_command_script(char), slack_email(char), customer_id(char), plan(char), webhook(text), ai_config(jsonb), large_file_storage(jsonb), git_sync(jsonb), default_app(char), default_scripts(jsonb), deploy_ui(jsonb), mute_critical_alerts(bool), color(char), operator_settings(jsonb), teams_command_script(text), teams_team_id(text), teams_team_name(text), git_app_installations(jsonb), ducklake(jsonb), slack_oauth_client_id(char), slack_oauth_client_secret(char), datatable(jsonb), teams_team_guid(text), auto_invite(jsonb), error_handler(jsonb), success_handler(jsonb), public_app_execution_limit_per_minute(int), dbt_warehouses(jsonb), guest_access_enabled(bool), guest_jwt_public_key(text), guest_jwt_jwks_url(text), ai_sessions_backup_generation(int) FK: (workspace_id) -> workspace(id) zombie_job_counter: job_id(uuid), counter(int) FK: (job_id) -> v2_job(id) diff --git a/backend/tests/ai_sessions.rs b/backend/tests/ai_sessions.rs new file mode 100644 index 0000000000..f76385d4e1 --- /dev/null +++ b/backend/tests/ai_sessions.rs @@ -0,0 +1,1085 @@ +//! The AI session backup routes (`/w/{w}/ai/sessions/*`): a browser pushes pieces of its +//! sessions into the workspace's object storage and pulls them back whole. Pinned against a +//! FilesystemStorage LFS so the test needs no object store, which also lets it read what +//! landed on disk: the objects must be ciphertext, since bucket credentials are shared far +//! more widely than a user's transcripts. +#![cfg(all(feature = "private", feature = "parquet"))] + +use serde_json::{json, Value}; +use sqlx::{Pool, Postgres}; +use windmill_common::utils::calculate_hash; +use windmill_test_utils::*; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder, token: &str) -> reqwest::RequestBuilder { + builder.header("Authorization", format!("Bearer {}", token)) +} + +async fn configure_primary_lfs(db: &Pool, root_path: &str) -> anyhow::Result<()> { + sqlx::query!( + "UPDATE workspace_settings SET large_file_storage = $1 WHERE workspace_id = $2", + json!({ + "type": "FilesystemStorage", + "root_path": root_path, + "public_resource": null, + "advanced_permissions": null + }), + "test-workspace" + ) + .execute(db) + .await?; + Ok(()) +} + +async fn list(base: &str, token: &str) -> anyhow::Result { + let resp = authed(client().get(format!("{base}/ai/sessions/list")), token) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + Ok(resp.json().await?) +} + +async fn pull(base: &str, token: &str, ids: &[&str]) -> anyhow::Result { + let resp = authed(client().post(format!("{base}/ai/sessions/pull")), token) + .json(&json!({ "ids": ids })) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + Ok(resp.json().await?) +} + +async fn push(base: &str, token: &str, body: Value) -> anyhow::Result { + Ok( + authed(client().post(format!("{base}/ai/sessions/push")), token) + .json(&body) + .send() + .await?, + ) +} + +fn copy_dir(from: &std::path::Path, to: &std::path::Path) -> std::io::Result<()> { + std::fs::create_dir_all(to)?; + for entry in std::fs::read_dir(from)? { + let entry = entry?; + let target = to.join(entry.file_name()); + if entry.path().is_dir() { + copy_dir(&entry.path(), &target)?; + } else { + std::fs::copy(entry.path(), target)?; + } + } + Ok(()) +} + +async fn rotate(base: &str, key: &str) -> anyhow::Result<()> { + let resp = authed( + client().post(format!("{base}/workspaces/encryption_key")), + "SECRET_TOKEN", + ) + .json(&json!({ "new_key": key })) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + Ok(()) +} + +/// The user's prefix on disk, `windmill_ai_sessions/{w_id}/g{generation}/{email hash}`, +/// under whichever generation is current. +fn user_root(storage_dir: &std::path::Path, email: &str) -> std::path::PathBuf { + let workspace = storage_dir.join("windmill_ai_sessions/test-workspace"); + let hash = calculate_hash(email); + std::fs::read_dir(&workspace) + .ok() + .into_iter() + .flatten() + .flatten() + .map(|entry| entry.path().join(&hash)) + .find(|path| path.exists()) + .expect("the user has backups under the current key") +} + +/// Every file under the storage root, as bytes. +fn files_under(root: &std::path::Path) -> Vec<(std::path::PathBuf, Vec)> { + let mut out = vec![]; + let mut stack = vec![root.to_path_buf()]; + while let Some(dir) = stack.pop() { + let Ok(entries) = std::fs::read_dir(&dir) else { + continue; + }; + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + stack.push(path); + } else { + out.push((path.clone(), std::fs::read(&path).unwrap_or_default())); + } + } + } + out +} + +#[sqlx::test(fixtures("base"))] +async fn test_backups_round_trip_encrypted_and_scoped_to_the_user( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let base = format!( + "http://localhost:{}/api/w/test-workspace", + server.addr.port() + ); + + // No storage configured: the browser is told to stop trying. + let listing = list(&base, "SECRET_TOKEN").await?; + assert_eq!(listing["enabled"], false); + assert_eq!(listing["sessions"], json!([])); + + let storage_dir = tempfile::tempdir()?; + configure_primary_lfs(&db, &storage_dir.path().to_string_lossy()).await?; + + let head = + json!({ "id": "s1", "workspace_id": "test-workspace", "createdAt": 1, "chatId": "c1" }); + let chat = json!({ "id": "c1", "sessionId": "s1", "title": "MARKER_PLAINTEXT_TITLE", "lastModified": 2, + "actualMessages": [], "displayMessages": [{"role": "user", "content": "hello"}] }); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ + "id": "s1", + "whole": true, + "head": head, + "chats": [{ "id": "c1", "record": chat }, { "id": "c2", "record": { "id": "c2" } }], + "images": [{ "chat_id": "c1", "id": "img1", "data_url": "data:image/png;base64,AAAA" }], + "artifacts": { "items": [], "versions": [] } + }] + }), + ) + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let pushed: Value = resp.json().await?; + assert_eq!(pushed["enabled"], true); + assert_eq!(pushed["results"], json!([{ "id": "s1" }])); + + let listing = list(&base, "SECRET_TOKEN").await?; + assert_eq!(listing["enabled"], true); + assert_eq!(listing["sessions"][0]["id"], "s1"); + + // A part more parts follow names its push, or the session would stay listed between + // the parts: one that does not is refused before anything of it lands. + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s1", "partial": true, "delete_chats": ["c1"] }] + }), + ) + .await?; + assert_eq!(resp.status(), 400, "{}", resp.text().await?); + assert_eq!( + list(&base, "SECRET_TOKEN").await?["sessions"][0]["id"], + "s1" + ); + assert_eq!( + pull(&base, "SECRET_TOKEN", &["s1"]).await?["sessions"][0]["chats"] + .as_array() + .unwrap() + .len(), + 2 + ); + + // A part with more of the session to follow lists nothing; the part that completes + // the push does, newest first. + let ids = |listing: &Value| -> Vec { + listing["sessions"] + .as_array() + .unwrap() + .iter() + .map(|s| s["id"].as_str().unwrap().to_string()) + .collect() + }; + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s2", "whole": true, "push": "p2", "opens": true, "head": { "id": "s2", "workspace_id": "test-workspace", "createdAt": 2, "chatId": "c" }, "partial": true }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + assert_eq!(ids(&list(&base, "SECRET_TOKEN").await?), vec!["s1"]); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s2", "whole": true, "push": "p2", "chats": [{ "id": "c", "record": { "id": "c" } }] }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + assert_eq!(ids(&list(&base, "SECRET_TOKEN").await?), vec!["s2", "s1"]); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "removed": ["s2"] }), + ) + .await?; + assert_eq!(resp.status(), 200); + + // A session that outgrew one answer (three chats of 12 MB against the 32 MB budget) + // comes in pages, each naming where the next picks up, and nothing is left out. + let big = "y".repeat(12 * 1024 * 1024); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s4", "whole": true, "push": "p4", "opens": true, "head": { "id": "s4", "workspace_id": "test-workspace", "createdAt": 4, "chatId": "c1" }, "partial": true }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + for cid in ["c1", "c2", "c3"] { + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s4", "whole": true, "push": "p4", "chats": [{ "id": cid, "record": { "id": cid, "big": big } }], "partial": cid != "c3" }] + }), + ) + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + } + let mut pages = vec![]; + let mut resume = json!(null); + loop { + let body = if resume.is_null() { + json!({ "ids": ["s4"] }) + } else { + json!({ "ids": ["s4"], "resume": resume }) + }; + let resp = authed( + client().post(format!("{base}/ai/sessions/pull")), + "SECRET_TOKEN", + ) + .json(&body) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let pulled: Value = resp.json().await?; + let page = pulled["sessions"][0].clone(); + assert_eq!(page["id"], "s4"); + resume = page["next"].clone(); + pages.push(page); + if resume.is_null() { + break; + } + assert!(pages.len() < 5, "a paged pull must end"); + } + assert!(pages.len() >= 2, "36 MB must not fit one answer"); + // Every page of an unchanged backup carries the same listing fingerprint; a chat added + // to the session changes it, which is what tells a browser its pages do not belong + // together any more. + let listing = pages[0]["listing"].clone(); + assert!(listing.is_string()); + assert!(pages.iter().all(|p| p["listing"] == listing)); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s4", "chats": [{ "id": "c0", "record": { "id": "c0", "n": 1 } }] }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let pulled = pull(&base, "SECRET_TOKEN", &["s4"]).await?; + assert_ne!(pulled["sessions"][0]["listing"], listing); + // So does a chat rewritten at the same size: the fingerprint takes in the entity tag, + // not only the size and a modification time the store may report coarsely. + let listing = pulled["sessions"][0]["listing"].clone(); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s4", "chats": [{ "id": "c0", "record": { "id": "c0", "n": 2 } }] }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let pulled = pull(&base, "SECRET_TOKEN", &["s4"]).await?; + assert_ne!(pulled["sessions"][0]["listing"], listing); + let mut chat_ids: Vec = pages + .iter() + .flat_map(|p| p["chats"].as_array().unwrap().iter()) + .map(|c| c["id"].as_str().unwrap().to_string()) + .collect(); + chat_ids.sort(); + assert_eq!(chat_ids, vec!["c1", "c2", "c3"]); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "removed": ["s4"] }), + ) + .await?; + assert_eq!(resp.status(), 200); + + // More objects than a page keeps listing metadata for, from a store that lists in no + // order: the pages still carry every one of them, each once. + let many = 5001; + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s5", "whole": true, "push": "p5", "opens": true, "head": { "id": "s5", "workspace_id": "test-workspace", "createdAt": 5, "chatId": "c00000" }, "partial": true }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + for start in (0..many).step_by(100) { + let chats: Vec = (start..(start + 100).min(many)) + .map(|i| json!({ "id": format!("c{i:05}"), "record": { "id": format!("c{i:05}") } })) + .collect(); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s5", "whole": true, "push": "p5", "chats": chats, "partial": start + 100 < many }] + }), + ) + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + } + let mut seen = std::collections::HashSet::new(); + let mut resume = json!(null); + let mut pages = 0; + loop { + let body = if resume.is_null() { + json!({ "ids": ["s5"] }) + } else { + json!({ "ids": ["s5"], "resume": resume }) + }; + let resp = authed( + client().post(format!("{base}/ai/sessions/pull")), + "SECRET_TOKEN", + ) + .json(&body) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let pulled: Value = resp.json().await?; + let page = &pulled["sessions"][0]; + for c in page["chats"].as_array().unwrap() { + assert!( + seen.insert(c["id"].as_str().unwrap().to_string()), + "a chat came twice" + ); + } + pages += 1; + resume = page["next"].clone(); + if resume.is_null() { + break; + } + assert!(pages < 5, "a paged pull must end"); + } + assert_eq!(pages, 2); + assert_eq!(seen.len(), many); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "removed": ["s5"] }), + ) + .await?; + assert_eq!(resp.status(), 200); + + // A head at exactly its cap round-trips: the ciphertext read back is a block larger. + let mut big_head = json!({ "id": "s3", "workspace_id": "test-workspace", "createdAt": 3, "chatId": "c", "pad": "" }); + let pad = 1024 * 1024 - serde_json::to_string(&big_head)?.len(); + big_head["pad"] = json!("x".repeat(pad)); + assert_eq!(serde_json::to_string(&big_head)?.len(), 1024 * 1024); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "sessions": [{ "id": "s3", "whole": true, "head": big_head }] }), + ) + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let pulled = pull(&base, "SECRET_TOKEN", &["s3"]).await?; + assert_eq!(pulled["sessions"][0]["head"], big_head); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "removed": ["s3"] }), + ) + .await?; + assert_eq!(resp.status(), 200); + + // An object larger than any push writes, planted with the bucket's credentials at a + // predictable key, is not read. + let planted = + user_root(storage_dir.path(), "test@windmill.dev").join("sessions/planted/head.json"); + std::fs::create_dir_all(planted.parent().unwrap())?; + std::fs::File::create(&planted)?.set_len(32 * 1024 * 1024 + 1)?; + let pulled = pull(&base, "SECRET_TOKEN", &["planted"]).await?; + assert_eq!(pulled["sessions"], json!([])); + std::fs::remove_dir_all(planted.parent().unwrap())?; + // Under a session that exists, a planted chat is skipped without buffering and without + // the page ending before it, so the pull neither balloons nor loops. + let planted_chat = + user_root(storage_dir.path(), "test@windmill.dev").join("sessions/s1/chats/planted.json"); + std::fs::File::create(&planted_chat)?.set_len(32 * 1024 * 1024 + 1)?; + let mut resume = json!(null); + let mut pages = 0; + let mut chat_ids = vec![]; + loop { + let body = if resume.is_null() { + json!({ "ids": ["s1"] }) + } else { + json!({ "ids": ["s1"], "resume": resume }) + }; + let resp = authed( + client().post(format!("{base}/ai/sessions/pull")), + "SECRET_TOKEN", + ) + .json(&body) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let pulled: Value = resp.json().await?; + assert_eq!(pulled["sessions"][0]["head"], head); + for c in pulled["sessions"][0]["chats"].as_array().unwrap() { + chat_ids.push(c["id"].as_str().unwrap().to_string()); + } + pages += 1; + resume = pulled["sessions"][0]["next"].clone(); + if resume.is_null() { + break; + } + assert!(pages < 5, "a planted object must not keep the pull going"); + } + chat_ids.sort(); + assert_eq!(chat_ids, vec!["c1", "c2"]); + std::fs::remove_file(&planted_chat)?; + + let pulled = pull(&base, "SECRET_TOKEN", &["s1", "never-pushed"]).await?; + assert_eq!(pulled["deferred"], json!([])); + let sessions = pulled["sessions"].as_array().unwrap(); + assert_eq!(sessions.len(), 1, "an id with no backup is simply absent"); + let s1 = &sessions[0]; + assert_eq!(s1["head"], head); + let mut chats = s1["chats"].as_array().unwrap().clone(); + chats.sort_by_key(|c| c["id"].as_str().unwrap().to_string()); + assert_eq!(chats[0]["record"], chat); + assert_eq!(chats[1]["id"], "c2"); + assert_eq!( + s1["images"], + json!([{ "chat_id": "c1", "id": "img1", "data_url": "data:image/png;base64,AAAA" }]) + ); + assert_eq!(s1["artifacts"], json!({ "items": [], "versions": [] })); + + // Nothing on disk carries the transcript in the clear. + let files = files_under(storage_dir.path()); + assert!( + files.len() >= 4, + "expected the pushed objects on disk, got {files:?}" + ); + for (path, bytes) in &files { + let text = String::from_utf8_lossy(bytes); + assert!( + !text.contains("MARKER_PLAINTEXT_TITLE") && !text.contains("base64,AAAA"), + "{} holds plaintext", + path.display() + ); + } + let key_paths: Vec = files + .iter() + .map(|(p, _)| { + p.strip_prefix(storage_dir.path()) + .unwrap() + .to_string_lossy() + .to_string() + }) + .collect(); + assert!( + key_paths + .iter() + .all(|p| p.starts_with("windmill_ai_sessions/test-workspace/") + && !p.contains("test@windmill.dev")), + "keys carry the workspace and never the email: {key_paths:?}" + ); + + // Another member of the workspace sees none of it. + let other = list(&base, "SECRET_TOKEN_2").await?; + assert_eq!(other["enabled"], true); + assert_eq!(other["sessions"], json!([])); + let other = pull(&base, "SECRET_TOKEN_2", &["s1"]).await?; + assert_eq!(other["sessions"], json!([])); + + // Nor after copying the first user's ciphertext under their own prefix, which anyone + // holding the bucket credentials can do: the key is bound to the user, not the workspace. + let first = user_root(storage_dir.path(), "test@windmill.dev"); + let second = first + .parent() + .unwrap() + .join(calculate_hash("test2@windmill.dev")); + copy_dir(&first, &second)?; + let other = pull(&base, "SECRET_TOKEN_2", &["s1"]).await?; + assert_eq!( + other["sessions"], + json!([]), + "relocated ciphertext must not decrypt for another user" + ); + std::fs::remove_dir_all(&second)?; + + // Deleting a chat takes its images along; a head-only push leaves the rest in place. + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s1", "delete_chats": ["c1"] }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let pulled = pull(&base, "SECRET_TOKEN", &["s1"]).await?; + let s1 = &pulled["sessions"][0]; + assert_eq!(s1["head"], head); + assert_eq!(s1["chats"].as_array().unwrap().len(), 1); + assert_eq!(s1["chats"][0]["id"], "c2"); + assert_eq!(s1["images"], json!([])); + + // Rotating the workspace key moves the routes to a fresh generation's prefix and deletes + // the older ones off the request rather than re-key anything; the answers name the new + // generation (`backup_generation`), which is what makes every browser push its sessions + // whole again, while `storage_id` names the storage and stays. + let before = list(&base, "SECRET_TOKEN").await?; + rotate(&base, &"b".repeat(64)).await?; + for _ in 0..100 { + if files_under(storage_dir.path()).is_empty() { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + assert!( + files_under(storage_dir.path()).is_empty(), + "a rotation must leave no backup object behind" + ); + let listing = list(&base, "SECRET_TOKEN").await?; + assert_eq!(listing["sessions"], json!([])); + assert_eq!(listing["storage_id"], before["storage_id"]); + assert_ne!( + listing["backup_generation"], before["backup_generation"], + "a rotation must bump the backup generation" + ); + assert_eq!( + pull(&base, "SECRET_TOKEN", &["s1"]).await?["sessions"], + json!([]) + ); + // The browser's next push fills the storage back under the new key. + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s1", "whole": true, "head": head, "chats": [{ "id": "c2", "record": { "id": "c2" } }] }] + }), + ) + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let pulled = pull(&base, "SECRET_TOKEN", &["s1"]).await?; + assert_eq!(pulled["sessions"][0]["head"], head); + assert_eq!(pulled["sessions"][0]["chats"][0]["id"], "c2"); + // Setting the key already in place is not a rotation the browsers would notice, so it + // keeps the backups. + let same = list(&base, "SECRET_TOKEN").await?; + rotate(&base, &"b".repeat(64)).await?; + let listing = list(&base, "SECRET_TOKEN").await?; + assert_eq!(listing["storage_id"], same["storage_id"]); + assert_eq!(listing["backup_generation"], same["backup_generation"]); + assert_eq!(listing["sessions"][0]["id"], "s1"); + assert_eq!( + pull(&base, "SECRET_TOKEN", &["s1"]).await?["sessions"][0]["head"], + head + ); + + // A push that does not open the session whole rides on the head in the storage; with + // none there (another device removed the backup, or nothing was ever pushed) it is + // refused and lists nothing, head or no head on it, until the session goes whole. + let s6_head = + json!({ "id": "s6", "workspace_id": "test-workspace", "createdAt": 6, "chatId": "c" }); + let not_listed = |listing: Value| { + listing["sessions"] + .as_array() + .unwrap() + .iter() + .all(|s| s["id"] != "s6") + }; + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s6", "chats": [{ "id": "c", "record": { "id": "c" } }] }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let answer: Value = resp.json().await?; + assert_eq!(answer["results"][0]["needs_whole"], true); + assert!(not_listed(list(&base, "SECRET_TOKEN").await?)); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s6", "whole": true, "head": s6_head, "chats": [{ "id": "c", "record": { "id": "c" } }] }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let answer: Value = resp.json().await?; + assert!(answer["results"][0]["needs_whole"].is_null()); + assert_eq!( + list(&base, "SECRET_TOKEN").await?["sessions"][0]["id"], + "s6" + ); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "removed": ["s6"] }), + ) + .await?; + assert_eq!(resp.status(), 200); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s6", "head": s6_head, "chats": [{ "id": "c2", "record": { "id": "c2" } }] }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let answer: Value = resp.json().await?; + assert_eq!(answer["results"][0]["needs_whole"], true); + assert!(not_listed(list(&base, "SECRET_TOKEN").await?)); + assert!( + files_under(&user_root(storage_dir.path(), "test@windmill.dev").join("sessions/s6")) + .is_empty() + ); + + // Between the parts of a whole push (head landed, marker not yet), an incremental push + // from another device is refused too: it rides on a listed session, and there is none + // until the last part, which lists it. + let s8_head = + json!({ "id": "s8", "workspace_id": "test-workspace", "createdAt": 8, "chatId": "c1" }); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s8", "whole": true, "push": "p8", "opens": true, "head": s8_head, "partial": true }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s8", "chats": [{ "id": "c9", "record": { "id": "c9" } }] }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let answer: Value = resp.json().await?; + assert_eq!(answer["results"][0]["needs_whole"], true); + let listed = |listing: Value| { + listing["sessions"] + .as_array() + .unwrap() + .iter() + .any(|s| s["id"] == "s8") + }; + assert!(!listed(list(&base, "SECRET_TOKEN").await?)); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s8", "whole": true, "push": "p8", "chats": [{ "id": "c1", "record": { "id": "c1" } }] }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + assert!(listed(list(&base, "SECRET_TOKEN").await?)); + assert_eq!( + pull(&base, "SECRET_TOKEN", &["s8"]).await?["sessions"][0]["chats"] + .as_array() + .unwrap() + .iter() + .map(|c| c["id"].as_str().unwrap().to_string()) + .collect::>(), + vec!["c1"] + ); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "removed": ["s8"] }), + ) + .await?; + assert_eq!(resp.status(), 200); + + // A whole push replaces the backup: what the storage held of the session and the new + // push does not carry (a chat deleted while the workspace was on another storage) goes. + let s9_head = + json!({ "id": "s9", "workspace_id": "test-workspace", "createdAt": 9, "chatId": "c1" }); + let s9_chats = |ids: &[&str]| -> Vec { + ids.iter() + .map(|c| json!({ "id": c, "record": { "id": c } })) + .collect() + }; + let pulled_chats = |pulled: Value| -> Vec { + pulled["sessions"][0]["chats"] + .as_array() + .unwrap() + .iter() + .map(|c| c["id"].as_str().unwrap().to_string()) + .collect() + }; + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s9", "whole": true, "head": s9_head, "chats": s9_chats(&["c1", "c2"]) }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + assert_eq!( + pulled_chats(pull(&base, "SECRET_TOKEN", &["s9"]).await?), + vec!["c1", "c2"] + ); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s9", "whole": true, "epoch": 1, "head": s9_head, "chats": s9_chats(&["c1"]) }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + assert_eq!( + pulled_chats(pull(&base, "SECRET_TOKEN", &["s9"]).await?), + vec!["c1"] + ); + // The marker carries the move count the push named, once; an incremental push at + // another count rides on nothing, one at the same count lands. + let s9_epochs = |listing: Value| -> Vec { + listing["sessions"] + .as_array() + .unwrap() + .iter() + .filter(|s| s["id"] == "s9") + .map(|s| s["epoch"].clone()) + .collect() + }; + assert_eq!( + s9_epochs(list(&base, "SECRET_TOKEN").await?), + vec![json!(1)] + ); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s9", "chats": s9_chats(&["c7"]) }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let answer: Value = resp.json().await?; + assert_eq!(answer["results"][0]["needs_whole"], true); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s9", "epoch": 1, "chats": s9_chats(&["c7"]) }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + assert_eq!( + pulled_chats(pull(&base, "SECRET_TOKEN", &["s9"]).await?), + vec!["c1", "c7"] + ); + + // An incremental push split over parts unlists the session while it is in progress (a + // pull between two parts would take a mix of old and new pieces for the backup) and + // lists it again with the last part; while one is in progress or abandoned, a push that + // is not part of it is refused, so the browser's next push of the session goes whole. + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s9", "epoch": 1, "push": "i1", "opens": true, "chats": s9_chats(&["c8"]), "partial": true }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + assert_eq!( + s9_epochs(list(&base, "SECRET_TOKEN").await?), + Vec::::new() + ); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s9", "epoch": 1, "chats": s9_chats(&["c11"]) }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let answer: Value = resp.json().await?; + assert_eq!(answer["results"][0]["needs_whole"], true); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s9", "epoch": 1, "push": "i1", "chats": s9_chats(&["c9"]) }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + assert_eq!( + s9_epochs(list(&base, "SECRET_TOKEN").await?), + vec![json!(1)] + ); + assert_eq!( + pulled_chats(pull(&base, "SECRET_TOKEN", &["s9"]).await?), + vec!["c1", "c7", "c8", "c9"] + ); + assert_eq!( + s9_epochs(list(&base, "SECRET_TOKEN").await?), + vec![json!(1)] + ); + + // Two devices pushing the session whole at once: the push that opened later replaced + // the earlier one's pieces, so the earlier one's last part is refused and lists nothing, + // and the session is listed with the later push's pieces alone. + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s9", "whole": true, "push": "t3", "opens": true, "head": s9_head, "chats": s9_chats(&["c3"]), "partial": true }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s9", "whole": true, "push": "t4", "opens": true, "head": s9_head, "chats": s9_chats(&["c4"]), "partial": true }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s9", "whole": true, "push": "t3", "chats": s9_chats(&["c5"]) }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let answer: Value = resp.json().await?; + assert_eq!(answer["results"][0]["needs_whole"], true); + assert!(list(&base, "SECRET_TOKEN").await?["sessions"] + .as_array() + .unwrap() + .iter() + .all(|s| s["id"] != "s9")); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s9", "whole": true, "push": "t4", "chats": s9_chats(&["c6"]) }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + assert_eq!( + pulled_chats(pull(&base, "SECRET_TOKEN", &["s9"]).await?), + vec!["c4", "c6"] + ); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "removed": ["s9"] }), + ) + .await?; + assert_eq!(resp.status(), 200); + + // Removal empties both prefixes. + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "removed": ["s1"] }), + ) + .await?; + assert_eq!(resp.status(), 200); + let listing = list(&base, "SECRET_TOKEN").await?; + assert_eq!(listing["sessions"], json!([])); + assert!( + listing["storage_id"].is_string(), + "an answer names its storage: {listing}" + ); + assert!( + files_under(storage_dir.path()).is_empty(), + "removal must leave no object behind" + ); + + Ok(()) +} + +#[sqlx::test(fixtures("base", "jobs_read_auth"))] +async fn test_backup_writes_are_refused_for_the_wrong_owner_token_or_id( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let base = format!( + "http://localhost:{}/api/w/test-workspace", + server.addr.port() + ); + let storage_dir = tempfile::tempdir()?; + configure_primary_lfs(&db, &storage_dir.path().to_string_lossy()).await?; + + // A push prepared for another user must not land under the caller's prefix. + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test2@windmill.dev", "sessions": [{ "id": "s1", "head": { "id": "s1" } }] }), + ) + .await?; + assert_eq!(resp.status(), 409, "{}", resp.text().await?); + + // Ids are what the server builds keys from. + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "sessions": [{ "id": "../s1", "head": { "id": "../s1" } }] }), + ) + .await?; + assert_eq!(resp.status(), 400, "{}", resp.text().await?); + + // A whole push opens with its head; one without is refused before anything of it lands, + // and nothing lists the session. + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "sessions": [{ "id": "s7", "whole": true, "chats": [{ "id": "c", "record": { "id": "c" } }] }] }), + ) + .await?; + assert_eq!(resp.status(), 400, "{}", resp.text().await?); + assert!(list(&base, "SECRET_TOKEN").await?["sessions"] + .as_array() + .unwrap() + .iter() + .all(|s| s["id"] != "s7")); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "removed": ["a/b"] }), + ) + .await?; + assert_eq!(resp.status(), 400, "{}", resp.text().await?); + + // An image is a base64 data URL, stored and served verbatim; anything JSON would have + // to escape (and so inflate past the pull budget) is refused. + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s1", "images": [{ "chat_id": "c1", "id": "i1", "data_url": "data:image/png;base64,\u{0001}\u{0001}\"" }] }] + }), + ) + .await?; + assert_eq!(resp.status(), 400, "{}", resp.text().await?); + + // Nested lists are bounded too: each entry is an object-store call. + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s1", "delete_chats": (0..1001).map(|i| format!("c{i}")).collect::>() }] + }), + ) + .await?; + assert_eq!(resp.status(), 400, "{}", resp.text().await?); + + // ...and across the whole request, not only per entry. + let sessions: Vec = (0..100) + .map(|i| { + json!({ "id": format!("s{i}"), "delete_chats": (0..50).map(|j| format!("c{j}")).collect::>() }) + }) + .collect(); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "sessions": sessions }), + ) + .await?; + assert_eq!(resp.status(), 400, "{}", resp.text().await?); + + // A pull body is a handful of ids; a large one is refused before it is parsed. + let resp = authed( + client().post(format!("{base}/ai/sessions/pull")), + "SECRET_TOKEN", + ) + .header("Content-Type", "application/json") + .body(format!("{{\"ids\":[\"{}\"]}}", "a".repeat(100_000))) + .send() + .await?; + assert_eq!(resp.status(), 413, "{}", resp.text().await?); + + // A scoped token (here `jobs:read`) is minted for something narrower than the user's + // whole assistant history. + let resp = authed( + client().get(format!("{base}/ai/sessions/list")), + "SCOPED_DENO_TOKEN", + ) + .send() + .await?; + assert_eq!(resp.status(), 403, "{}", resp.text().await?); + + assert!(files_under(storage_dir.path()).is_empty()); + Ok(()) +} diff --git a/backend/windmill-api-workspaces/src/ai_session_backups.rs b/backend/windmill-api-workspaces/src/ai_session_backups.rs new file mode 100644 index 0000000000..bbdbfbc8c9 --- /dev/null +++ b/backend/windmill-api-workspaces/src/ai_session_backups.rs @@ -0,0 +1,150 @@ +//! What the workspace key rotation and the AI session backup routes +//! (`windmill-api/src/ai_sessions.rs`) share about the backups in the workspace storage. +//! +//! The backups are ciphertext under the workspace key and live under a prefix named by a +//! generation the rotation bumps (`workspace_settings.ai_sessions_backup_generation`) in the +//! transaction that commits the new key. A rotation does not re-key them: once committed, +//! the routes read and write under the new generation's prefix and answer with its number +//! (`backup_generation`; the storage identity, `storage_id`, names the storage and does not +//! change), so every browser marks its sync state stale and pushes its sessions whole again +//! there, and every older generation, which nothing writes to any +//! more, is deleted off the request at leisure. Sessions no browser holds any more are lost, +//! which a rotation (a rare operation) accepts in exchange for having no key but the current +//! one to read with and nothing to rewrite in place. A generation is never reused, so no +//! deletion, however late, can touch live objects; a rotation that fails before its commit +//! bumps nothing and deletes nothing; two rotations racing serialize on the key row. + +use std::sync::Arc; + +use futures::TryStreamExt; +use windmill_common::error::{Error, Result}; +use windmill_common::utils::calculate_hash; +use windmill_common::DB; +use windmill_object_store::object_store_reexports::{ + ObjectStore, ObjectStoreError, Path as ObjectPath, +}; +use windmill_object_store::{object_store_error_to_error, ObjectStoreResource}; +use windmill_types::s3::LargeFileStorage; + +/// The root of every AI session backup key in a workspace's storage. +pub const ROOT: &str = "windmill_ai_sessions"; +/// The push body cap: no object written through the routes is larger. One that is was +/// planted by whoever holds the bucket's credentials, and is left unread. +pub const MAX_OBJECT_BYTES: usize = 32 * 1024 * 1024; + +const IO_CONCURRENCY: usize = 8; + +/// The prefix of one generation's objects: `windmill_ai_sessions/{w_id}/g{generation}/`. +pub fn generation_prefix(w_id: &str, generation: i64) -> String { + format!("{ROOT}/{w_id}/g{generation}") +} + +/// Names the storage the backups are in, by what locates its objects (endpoint, region, +/// bucket; never the credentials, which rotate), so a browser tells that its sync state was +/// recorded against another storage; the generation, answered alongside, tells it a +/// rotation happened in this one. +pub fn storage_id(resource: &ObjectStoreResource) -> String { + let location = match resource { + ObjectStoreResource::S3(s) => format!( + "s3:{}:{}:{}:{}", + s.endpoint, + s.port.unwrap_or_default(), + s.region, + s.bucket + ), + ObjectStoreResource::Azure(a) => format!( + "azure:{}:{}:{}", + a.endpoint.as_deref().unwrap_or_default(), + a.account_name, + a.container_name + ), + ObjectStoreResource::Gcs(g) => format!("gcs:{}", g.bucket), + ObjectStoreResource::Filesystem(f) => format!("fs:{}", f.root_path), + }; + calculate_hash(&location)[..16].to_string() +} + +/// The workspace's primary storage, resolved without a caller: a rotation runs the +/// deletion off its own request. +async fn primary_store(db: &DB, w_id: &str) -> Result>> { + let Some(lfs_json) = sqlx::query_scalar!( + "SELECT large_file_storage FROM workspace_settings WHERE workspace_id = $1", + w_id + ) + .fetch_optional(db) + .await? + .flatten() else { + return Ok(None); + }; + let lfs: LargeFileStorage = serde_json::from_value(lfs_json) + .map_err(|e| Error::internal_err(format!("parsing large_file_storage: {e}")))?; + let resource_value = if matches!(lfs, LargeFileStorage::FilesystemStorage(_)) { + serde_json::Value::Null + } else { + let path = lfs.get_s3_resource_path(); + let path = path.strip_prefix("$res:").unwrap_or(path); + windmill_common::workspaces::transform_json_value_unchecked( + &serde_json::Value::String(format!("$res:{path}")), + w_id, + db, + ) + .await? + }; + let resource = windmill_object_store::lfs_to_object_store_resource(&lfs, resource_value)?; + Ok(Some( + windmill_object_store::build_object_store_client(&resource).await?, + )) +} + +/// The generation an object key sits under, `None` for a key of no generation (an older +/// layout), which counts as older than any. +fn generation_of(w_id: &str, key: &ObjectPath) -> Option { + key.as_ref() + .strip_prefix(&format!("{ROOT}/{w_id}/g"))? + .split('/') + .next()? + .parse() + .ok() +} + +/// Deletes, off the request and as the listing streams, every object of the workspace's +/// backups from a generation older than `current`, once the rotation that made `current` +/// the generation has committed: nothing writes there any more but a push that resolved its +/// prefix before the commit, junk the browser's next push of that session rewrites under the +/// current prefix, as is anything a deletion cut short left behind. For the rotation route, +/// which authorized its caller as a superadmin. +pub(crate) fn spawn_delete_older(db: DB, w_id: String, current: i64) { + tokio::spawn(async move { + let store = match primary_store(&db, &w_id).await { + Ok(Some(store)) => store, + Ok(None) => return, + Err(e) => { + tracing::warn!("older AI session backups of {w_id} left in place: {e:#}"); + return; + } + }; + let prefix = ObjectPath::from(format!("{ROOT}/{w_id}")); + let deleted = store + .list(Some(&prefix)) + .map_err(object_store_error_to_error) + .try_for_each_concurrent(IO_CONCURRENCY, |meta| { + let (store, w_id) = (&store, &w_id); + async move { + if generation_of(w_id, &meta.location).is_some_and(|g| g >= current) { + return Ok(()); + } + match store.delete(&meta.location).await { + Ok(()) | Err(ObjectStoreError::NotFound { .. }) => Ok(()), + Err(e) => Err(object_store_error_to_error(e)), + } + } + }) + .await; + match deleted { + Ok(()) => { + tracing::info!("deleted the AI session backups of {w_id} older than g{current}") + } + Err(e) => tracing::warn!("deleting the older AI session backups of {w_id}: {e:#}"), + } + }); +} diff --git a/backend/windmill-api-workspaces/src/lib.rs b/backend/windmill-api-workspaces/src/lib.rs index 017c9702a5..22f2a2c5bb 100644 --- a/backend/windmill-api-workspaces/src/lib.rs +++ b/backend/windmill-api-workspaces/src/lib.rs @@ -1,6 +1,8 @@ +#[cfg(feature = "parquet")] +pub mod ai_session_backups; +pub mod data_metrics; pub mod datatable_migrations; pub mod deployment_requests; -pub mod data_metrics; pub mod workspaces; pub mod workspaces_extra; pub mod workspaces_oss; diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 6048772ff6..1a3c24f096 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -5339,6 +5339,17 @@ async fn set_encryption_key( let mut tx = db.begin().await?; + // Under the row's lock, so two rotations racing serialize and each sees the key the + // other committed. The AI session backups in the workspace storage live under a prefix + // named by a generation this bumps (with the key, in this transaction) rather than + // being re-keyed; the older generations are deleted once this one has committed (see + // `ai_session_backups`). The same key set again is no rotation to them. + let previous_key: String = sqlx::query_scalar( + "SELECT key FROM workspace_key WHERE workspace_id = $1 AND kind = 'cloud' FOR UPDATE", + ) + .bind(&w_id) + .fetch_one(&mut *tx) + .await?; sqlx::query!( "UPDATE workspace_key SET key = $1 WHERE workspace_id = $2", request.new_key.clone(), @@ -5346,6 +5357,18 @@ async fn set_encryption_key( ) .execute(&mut *tx) .await?; + let backups_generation: Option = if previous_key != request.new_key { + sqlx::query_scalar( + "UPDATE workspace_settings SET ai_sessions_backup_generation = \ + ai_sessions_backup_generation + 1 WHERE workspace_id = $1 \ + RETURNING ai_sessions_backup_generation", + ) + .bind(&w_id) + .fetch_optional(&mut *tx) + .await? + } else { + None + }; let mut reencrypted_secret_paths: Vec = Vec::new(); if !request.skip_reencrypt.unwrap_or(false) { @@ -5402,6 +5425,15 @@ async fn set_encryption_key( // Invalidate the cache only after the transaction has committed WORKSPACE_CRYPT_CACHE.remove(w_id.as_str()); + // Nothing writes under the older generations any more; the browsers push their + // sessions again under the new one. + #[cfg(feature = "parquet")] + if let Some(generation) = backups_generation { + crate::ai_session_backups::spawn_delete_older(db.clone(), w_id.clone(), generation); + } + #[cfg(not(feature = "parquet"))] + let _ = backups_generation; + // Build the batch: one event for the encryption key itself plus one per // re-encrypted secret variable. The batch entrypoint dispatches a single // git-sync job per repo carrying all items, so repos with Secrets sync diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 500504a663..9f742a129b 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -13195,6 +13195,153 @@ paths: type: boolean description: more buckets matched than were returned, so summing them under-reports + /w/{workspace}/ai/sessions/list: + get: + summary: list the calling user's AI session backups in the workspace object storage + operationId: listAiSessionBackups + tags: + - ai + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: backups, newest first; `enabled` is false when the workspace has no storage for them + content: + application/json: + schema: + type: object + required: + - enabled + - sessions + properties: + enabled: + type: boolean + storage_id: + type: string + description: names the storage answered from; sync state recorded against another one is void + backup_generation: + type: integer + description: bumped by every workspace key rotation; sync state recorded under another one is void + sessions: + type: array + description: the newest 500 at most + items: + $ref: "#/components/schemas/AISessionBackupListing" + truncated: + type: boolean + description: the user has more sessions than the answer names + + /w/{workspace}/ai/sessions/pull: + post: + summary: fetch whole AI session backups + operationId: pullAiSessionBackups + tags: + - ai + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - ids + properties: + ids: + type: array + maxItems: 20 + items: + type: string + resume: + $ref: "#/components/schemas/AISessionBackupCursor" + responses: + "200": + description: the backups found; `deferred` lists ids that did not fit the response budget + content: + application/json: + schema: + type: object + required: + - enabled + - sessions + - deferred + properties: + enabled: + type: boolean + storage_id: + type: string + backup_generation: + type: integer + sessions: + type: array + items: + $ref: "#/components/schemas/AISessionBackup" + deferred: + type: array + items: + type: string + + /w/{workspace}/ai/sessions/push: + post: + summary: write changed pieces of AI sessions to their backups, and remove deleted ones + operationId: pushAiSessionBackups + tags: + - ai + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - owner + properties: + owner: + type: string + description: the email the push was prepared for; refused with a 409 when it is not the caller's + sessions: + type: array + items: + $ref: "#/components/schemas/AISessionBackupPush" + removed: + type: array + items: + type: string + responses: + "200": + description: one result per session written or removed, in request order + content: + application/json: + schema: + type: object + required: + - enabled + - results + properties: + enabled: + type: boolean + storage_id: + type: string + backup_generation: + type: integer + results: + type: array + items: + type: object + required: + - id + properties: + id: + type: string + error: + type: string + needs_whole: + type: boolean + description: nothing was written and the session must be pushed whole again; an incremental part found no listed session to ride on (the backup was removed, or a push split over parts is in progress or was abandoned), or a later part of a push split over parts found another push had superseded it + /w/{workspace}/ai/shared_artifacts/share: post: summary: share an AI session artifact with the workspace @@ -28289,6 +28436,155 @@ components: fixes) from the workspace UI. Read from the workspace's own settings even when the providers served fall back to the instance config. AI agent steps and the AI sandbox in flows are unaffected. + sessions_storage_disabled: + type: boolean + description: >- + Stops browsers from backing their AI sessions up to the workspace's object + storage. Read from the workspace's own settings like `copilot_disabled`. + + AISessionBackupListing: + type: object + required: + - id + - updated_at + - epoch + properties: + id: + type: string + updated_at: + type: string + format: date-time + epoch: + type: integer + description: the session's move count when this copy was pushed; of a session two workspaces list, the copy with the higher one is the later + + AISessionBackupImage: + type: object + required: + - chat_id + - id + - data_url + properties: + chat_id: + type: string + id: + type: string + data_url: + type: string + + AISessionBackupChat: + type: object + required: + - id + - record + properties: + id: + type: string + record: + type: object + additionalProperties: true + + AISessionBackup: + type: object + required: + - id + - head + - chats + - images + - listing + properties: + id: + type: string + head: + type: object + additionalProperties: true + chats: + type: array + items: + $ref: "#/components/schemas/AISessionBackupChat" + images: + type: array + items: + $ref: "#/components/schemas/AISessionBackupImage" + artifacts: + type: object + additionalProperties: true + next: + $ref: "#/components/schemas/AISessionBackupCursor" + listing: + type: string + description: a fingerprint of the session's listing; pages of one session whose fingerprints differ do not belong together + moved: + type: boolean + description: the backup kept changing while this page was read, so it may mix two versions; the browser starts the session over + + AISessionBackupCursor: + type: object + description: where a pull of a session that did not fit one answer whole picks up; the rest of the session follows a pull naming that session alone with this as `resume` + required: + - id + - images + - after + properties: + id: + type: string + images: + type: boolean + after: + type: string + + AISessionBackupPush: + type: object + required: + - id + properties: + id: + type: string + head: + type: object + additionalProperties: true + chats: + type: array + items: + $ref: "#/components/schemas/AISessionBackupChat" + images: + type: array + items: + $ref: "#/components/schemas/AISessionBackupImage" + artifacts: + type: object + additionalProperties: true + delete_chats: + type: array + items: + type: string + delete_images: + type: array + items: + type: object + required: + - chat_id + - id + properties: + chat_id: + type: string + id: + type: string + partial: + type: boolean + description: more parts of this session follow, in this push or a later one; the session is not listed on this one. Such a part names its push (`push`), or it is refused + whole: + type: boolean + description: a part of a push of the session whole; the head is on the part that opens it, which replaces whatever the storage holds of the session, and every piece the browser has is on one of them. An incremental part instead rides on a session the storage lists and is refused with needs_whole when it lists none + push: + type: string + description: a push split over several parts names itself on each with a token the browser draws; the part that opens it unlists the session and the last part lists it again, and a later part is written only while that token is the one there (refused with needs_whole otherwise) + opens: + type: boolean + description: this part opens the push named by `push` + epoch: + type: integer + description: the session's move count (its record's `moves`), kept with the marker that lists the session; an incremental part rides on the marker of the same count FreeTierInfo: type: object diff --git a/backend/windmill-api/src/ai.rs b/backend/windmill-api/src/ai.rs index 8101064c3d..8b9c7929ab 100644 --- a/backend/windmill-api/src/ai.rs +++ b/backend/windmill-api/src/ai.rs @@ -451,6 +451,10 @@ pub struct AIConfig { /// and the AI sandbox are unaffected, so the providers stay in force. #[serde(default, skip_serializing_if = "std::ops::Not::not")] pub copilot_disabled: bool, + /// Stops browsers from backing their AI sessions up to the workspace's object storage + /// (`ai_sessions.rs`). Read from the workspace's own row like `copilot_disabled`. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub sessions_storage_disabled: bool, } /// Negotiated rates in USD per million tokens. An unset cache rate is read as the @@ -527,6 +531,9 @@ pub fn workspaced_service() -> Router { #[cfg(feature = "bedrock")] let router = router.route("/check_bedrock_credentials", get(check_bedrock_credentials)); + #[cfg(feature = "parquet")] + let router = router.nest("/sessions", crate::ai_sessions::workspaced_service()); + router } diff --git a/backend/windmill-api/src/ai_sessions.rs b/backend/windmill-api/src/ai_sessions.rs new file mode 100644 index 0000000000..fca9c35fca --- /dev/null +++ b/backend/windmill-api/src/ai_sessions.rs @@ -0,0 +1,1316 @@ +//! Lazily replicated backups of the browser's AI sessions in the workspace's object storage. +//! +//! The browser keeps the sessions in IndexedDB and pushes changed pieces here in batches; an +//! empty browser restores from what was pushed. The server owns the key layout, keeps the +//! caller's own prefix the only one it can reach, and encrypts every object with the +//! workspace key so bucket credentials do not read transcripts: +//! +//! ```text +//! windmill_ai_sessions/{w_id}/g{generation}/{sha256(email)}/sessions/{sid}/head.json +//! windmill_ai_sessions/{w_id}/g{generation}/{sha256(email)}/sessions/{sid}/chats/{cid}.json +//! windmill_ai_sessions/{w_id}/g{generation}/{sha256(email)}/sessions/{sid}/artifacts.json +//! windmill_ai_sessions/{w_id}/g{generation}/{sha256(email)}/images/{sid}/{cid}/{iid} +//! windmill_ai_sessions/{w_id}/g{generation}/{sha256(email)}/index/{sid}/{epoch} +//! ``` +//! +//! The index marker is empty, written last by every push of the session, and is what a +//! listing reads: one object per session, whatever the session holds, its `last_modified` +//! the session's `updated_at`. + +use crate::db::{ApiAuthed, DB}; +use axum::{ + extract::{DefaultBodyLimit, Path}, + routing::{get, post}, + Extension, Json, Router, +}; +use futures::{StreamExt, TryStreamExt}; +use magic_crypt::{MagicCrypt256, MagicCryptTrait}; +use serde::{Deserialize, Serialize}; +use serde_json::value::RawValue; +use std::sync::Arc; +use windmill_api_auth::is_effectively_unscoped; +use windmill_api_workspaces::ai_session_backups::{ + generation_prefix, storage_id, MAX_OBJECT_BYTES, +}; +use windmill_common::error::{Error, JsonResult, Result}; +use windmill_common::utils::calculate_hash; +use windmill_common::variables::{crypt_from_key_with_suffix, get_workspace_key}; +use windmill_object_store::object_store_reexports::{ + ObjectStore, ObjectStoreError, Path as ObjectPath, PutPayload, +}; +use windmill_object_store::{build_object_store_client, object_store_error_to_error}; + +const PUSH_BODY_LIMIT: usize = MAX_OBJECT_BYTES; +/// A pull names at most MAX_PULL_IDS ids of 64 bytes; anything larger is not a pull. +const PULL_BODY_LIMIT: usize = 64 * 1024; +/// A pull answer larger than this hands the remaining ids back as `deferred`. +const PULL_RESPONSE_BUDGET: usize = 32 * 1024 * 1024; +const MAX_HEAD_BYTES: usize = 1024 * 1024; +/// What the cipher adds to a plaintext at most (a block of padding): an object stored at a +/// cap is that much larger than the cap when read back. +const CIPHER_PADDING: usize = 16; +/// The browser bounds an image to a 1568 px edge and re-encodes past 700 KB; this is +/// well above what that produces. +const MAX_IMAGE_BYTES: usize = 4 * 1024 * 1024; +const MAX_PULL_IDS: usize = 20; +const MAX_PUSH_SESSIONS: usize = 100; +const MAX_REMOVED: usize = 200; +const MAX_CHATS_PER_ENTRY: usize = 100; +const MAX_IMAGES_PER_ENTRY: usize = 500; +const MAX_DELETES_PER_ENTRY: usize = 1000; +const MAX_OPERATIONS_PER_PUSH: usize = 4000; +/// Entries of listing metadata a pull holds per page of a session. +const MAX_LISTED_OBJECTS: usize = 5000; +/// Session markers a listing scans, and the newest sessions it answers with. +const MAX_LIST_SCAN: usize = 50_000; +const LIST_MAX: usize = 500; +const IO_CONCURRENCY: usize = 8; + +pub fn workspaced_service() -> Router { + Router::new() + .route("/list", get(list)) + .route( + "/pull", + post(pull).layer(DefaultBodyLimit::max(PULL_BODY_LIMIT)), + ) + .route( + "/push", + post(push).layer(DefaultBodyLimit::max(PUSH_BODY_LIMIT)), + ) +} + +/// What reading an object yields. `Gone`: not there (deleted since the listing, or never +/// pushed). `Grown`: larger than expected, so replaced since the listing (or planted), and +/// left unread; a pull answers with a page ending before it rather than without it. +/// `Foreign`: it does not decrypt for this user (written under another user's or +/// workspace's key), and must not take the rest of the session down with it. +enum Read { + Text(String), + Gone, + Grown, + Foreign, +} + +/// The user's prefix in the workspace storage, plus what reads and writes it. +struct Backend { + store: Arc, + mc: MagicCrypt256, + prefix: String, + /// Name the storage and the generation the objects are under, for the browser's sync + /// state: a row recorded against another storage or generation is stale, a removal is + /// owed to the storage alone (a rotation deleted the older generation's copy anyway). + storage_id: String, + generation: i64, +} + +impl Backend { + fn index_prefix(&self) -> ObjectPath { + ObjectPath::from(format!("{}/index/", self.prefix)) + } + + /// The marker that lists the session, named by the session's move count so that of a + /// session two workspaces list, the copy moved last is told from the listing alone. + fn index_key(&self, sid: &str, epoch: u32) -> ObjectPath { + ObjectPath::from(format!("{}/index/{sid}/{epoch}", self.prefix)) + } + + fn index_session_prefix(&self, sid: &str) -> ObjectPath { + ObjectPath::from(format!("{}/index/{sid}/", self.prefix)) + } + + /// The token of the push split over parts in progress, under the session so a removal + /// or the next whole push clears it with the rest. + fn push_key(&self, sid: &str) -> ObjectPath { + ObjectPath::from(format!("{}/sessions/{sid}/push", self.prefix)) + } + + fn session_prefix(&self, sid: &str) -> ObjectPath { + ObjectPath::from(format!("{}/sessions/{sid}/", self.prefix)) + } + + fn head_key(&self, sid: &str) -> ObjectPath { + ObjectPath::from(format!("{}/sessions/{sid}/head.json", self.prefix)) + } + + fn chat_key(&self, sid: &str, cid: &str) -> ObjectPath { + ObjectPath::from(format!("{}/sessions/{sid}/chats/{cid}.json", self.prefix)) + } + + fn artifacts_key(&self, sid: &str) -> ObjectPath { + ObjectPath::from(format!("{}/sessions/{sid}/artifacts.json", self.prefix)) + } + + fn images_prefix(&self, sid: &str) -> ObjectPath { + ObjectPath::from(format!("{}/images/{sid}/", self.prefix)) + } + + fn chat_images_prefix(&self, sid: &str, cid: &str) -> ObjectPath { + ObjectPath::from(format!("{}/images/{sid}/{cid}/", self.prefix)) + } + + fn image_key(&self, sid: &str, cid: &str, iid: &str) -> ObjectPath { + ObjectPath::from(format!("{}/images/{sid}/{cid}/{iid}", self.prefix)) + } + + fn seal(&self, plaintext: &[u8]) -> Vec { + self.mc.encrypt_bytes_to_bytes(plaintext) + } + + /// Bytes written. + async fn put_sealed(&self, key: &ObjectPath, ciphertext: Vec) -> Result { + let written = ciphertext.len(); + self.store + .put(key, PutPayload::from(ciphertext)) + .await + .map_err(object_store_error_to_error)?; + Ok(written) + } + + async fn put(&self, key: &ObjectPath, plaintext: &[u8]) -> Result { + self.put_sealed(key, self.seal(plaintext)).await + } + + /// `max` is what the listing said the object holds, or the cap of its kind for one read + /// without a listing: checked before buffering, since whoever holds the bucket's + /// credentials can put anything at a predictable key. + async fn get(&self, key: &ObjectPath, max: usize) -> Result { + let result = match self.store.get(key).await { + Ok(result) => result, + Err(ObjectStoreError::NotFound { .. }) => return Ok(Read::Gone), + Err(e) => return Err(object_store_error_to_error(e)), + }; + let size = result.meta.size as usize; + // Larger than any push writes: planted, whatever the listing said, and skipped like + // an object of another key rather than retried like one that grew. + if size > MAX_OBJECT_BYTES { + tracing::warn!("AI session backup object {key} is larger than any push writes"); + return Ok(Read::Foreign); + } + if size > max { + return Ok(Read::Grown); + } + let bytes = result.bytes().await.map_err(object_store_error_to_error)?; + // The objects are JSON and data URLs: a wrong key's output failing UTF-8 tells it + // apart beyond the cipher's padding check, which a wrong key passes now and then. + match self + .mc + .decrypt_bytes_to_bytes(&bytes) + .ok() + .and_then(|plaintext| String::from_utf8(plaintext).ok()) + { + Some(text) => Ok(Read::Text(text)), + None => { + tracing::warn!("AI session backup object {key} does not decrypt for its reader"); + Ok(Read::Foreign) + } + } + } + + async fn delete(&self, key: &ObjectPath) -> Result<()> { + match self.store.delete(key).await { + Ok(()) | Err(ObjectStoreError::NotFound { .. }) => Ok(()), + Err(e) => Err(object_store_error_to_error(e)), + } + } + + /// The entries under `prefix` past `after` in key order, as many as fit `budget` bytes; + /// `true` when more follow. Every key past `after` is seen and the MAX_LISTED_OBJECTS + /// smallest kept (a max-heap dropping its largest), since a page is defined by key + /// order and the store promises none; that cap is what bounds a pull's memory, a + /// session growing by valid pushes without limit. With `at_least_one`, the first entry + /// is taken whatever its size, so an answer owed the session makes progress on it (no + /// object exceeds the push body cap). + async fn list_within( + &self, + prefix: &ObjectPath, + after: Option<&ObjectPath>, + budget: usize, + at_least_one: bool, + ) -> Result<(Vec<(ObjectPath, usize)>, bool)> { + let mut kept: std::collections::BinaryHeap<(ObjectPath, usize)> = Default::default(); + let mut dropped = false; + let mut stream = match after { + Some(after) => self.store.list_with_offset(Some(prefix), after), + None => self.store.list(Some(prefix)), + }; + while let Some(meta) = stream.next().await { + let meta = meta.map_err(object_store_error_to_error)?; + kept.push((meta.location, meta.size as usize)); + if kept.len() > MAX_LISTED_OBJECTS { + kept.pop(); + dropped = true; + } + } + let mut entries = vec![]; + let mut total = 0; + for (key, size) in kept.into_sorted_vec() { + if total + size > budget && !(at_least_one && entries.is_empty()) { + return Ok((entries, true)); + } + total += size; + entries.push((key, size)); + } + Ok((entries, dropped)) + } + + /// Bytes written. Sealed up front so every stream item is owned: an item borrowing + /// from the request makes the future higher-ranked over that lifetime, which the + /// handler's `Send` bound cannot prove. + async fn put_all(&self, puts: Vec<(ObjectPath, Vec)>) -> Result { + futures::stream::iter(puts) + .map(|(key, ciphertext)| async move { self.put_sealed(&key, ciphertext).await }) + .buffer_unordered(IO_CONCURRENCY) + .try_fold(0, |acc, n| async move { Ok::<_, Error>(acc + n) }) + .await + } + + async fn delete_all(&self, keys: Vec) -> Result<()> { + futures::stream::iter(keys) + .map(|key| async move { self.delete(&key).await }) + .buffer_unordered(IO_CONCURRENCY) + .try_collect::>() + .await?; + Ok(()) + } + + /// A fingerprint of the session's marker and of everything listed under its two + /// prefixes (key, size, modification time, entity tag and version), combined as the + /// listing streams and in no particular order, so a session of any size costs bounded + /// memory. `None` for a session + /// the storage does not list. Taken before and after a page is read, so a page a push + /// changed under is read again; pages of one pull carry it, and the browser starts the + /// session over when it moved between two of them. + async fn listing_fingerprint(&self, sid: &str) -> Result> { + use std::hash::{DefaultHasher, Hash, Hasher}; + // The entity tag and version go in with the key, size and time: a store reports + // modification times coarsely, and an object rewritten at the same size within that + // grain would otherwise fingerprint the same. + fn fold( + acc: u64, + location: &str, + size: S, + modified: i64, + e_tag: Option<&str>, + version: Option<&str>, + ) -> u64 { + let mut hasher = DefaultHasher::new(); + (location, size, modified, e_tag, version).hash(&mut hasher); + acc.wrapping_add(hasher.finish()) + } + let mut acc = 0u64; + let mut listed = false; + for (marker, prefix) in [ + (true, self.index_session_prefix(sid)), + (false, self.session_prefix(sid)), + (false, self.images_prefix(sid)), + ] { + let mut stream = self.store.list(Some(&prefix)); + while let Some(meta) = stream.next().await { + let meta = meta.map_err(object_store_error_to_error)?; + listed |= marker; + acc = fold( + acc, + meta.location.as_ref(), + meta.size, + meta.last_modified.timestamp_millis(), + meta.e_tag.as_deref(), + meta.version.as_deref(), + ); + } + } + if !listed { + return Ok(None); + } + Ok(Some(format!("{acc:016x}"))) + } + + async fn exists(&self, key: &ObjectPath) -> Result { + match self.store.head(key).await { + Ok(_) => Ok(true), + Err(ObjectStoreError::NotFound { .. }) => Ok(false), + Err(e) => Err(object_store_error_to_error(e)), + } + } + + /// Deletes as the listing streams, so a prefix of any size costs bounded memory. + async fn delete_prefix(&self, prefix: &ObjectPath) -> Result<()> { + self.store + .list(Some(prefix)) + .map_err(object_store_error_to_error) + .try_for_each_concurrent(IO_CONCURRENCY, |meta| async move { + self.delete(&meta.location).await + }) + .await + } +} + +/// Backups are the user's own browser state and nothing else may reach them: a job token +/// may carry an `on_behalf_of` identity, and every scoped token (guest, embed, app policy, +/// MCP) is minted for something narrower than the user's whole assistant history. +fn require_plain_user_token(authed: &ApiAuthed) -> Result<()> { + if authed.job_id.is_some() || !is_effectively_unscoped(authed.scopes.as_deref()) { + return Err(Error::PermissionDenied( + "AI session backups are only reachable with an unscoped user token".to_string(), + )); + } + Ok(()) +} + +/// Every key is assembled server-side from ids the browser mints (`createLongHash` and +/// `randomUUID` forms), so anything outside this alphabet is a forged id, not a real one. +fn require_valid_id(kind: &str, id: &str) -> Result<()> { + let ok = !id.is_empty() + && id.len() <= 64 + && id + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'_' || b == b'-'); + if !ok { + return Err(Error::BadRequest(format!("invalid {kind} id: {id:?}"))); + } + Ok(()) +} + +/// Images travel as base64 data URLs and are stored verbatim, so they serialize back into +/// a pull answer at exactly their stored size; anything else (control characters, +/// quotes) could grow several times under JSON escaping and defeat the pull budget. +fn require_data_url(data_url: &str) -> Result<()> { + let ok = data_url.len() <= MAX_IMAGE_BYTES + && data_url + .strip_prefix("data:") + .and_then(|rest| rest.split_once(";base64,")) + .is_some_and(|(mime, payload)| { + !mime.is_empty() + && mime.bytes().all(|b| { + b.is_ascii_alphanumeric() || matches!(b, b'/' | b'.' | b'+' | b'-') + }) + && payload + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'+' | b'/' | b'=')) + }); + if !ok { + return Err(Error::BadRequest( + "an image must be a base64 data URL within the size cap".to_string(), + )); + } + Ok(()) +} + +fn require_json_object(kind: &str, raw: &RawValue, max_bytes: usize) -> Result<()> { + let text = raw.get(); + if !text.trim_start().starts_with('{') { + return Err(Error::BadRequest(format!("{kind} must be a JSON object"))); + } + if text.len() > max_bytes { + return Err(Error::BadRequest(format!( + "{kind} exceeds {max_bytes} bytes" + ))); + } + Ok(()) +} + +/// `None` when the workspace has nowhere to keep backups: no primary storage configured, or +/// the admin switched them off. Both read as `enabled: false` so the browser stops trying. +async fn backend(authed: &ApiAuthed, db: &DB, w_id: &str) -> Result> { + let (disabled, generation) = sqlx::query_as::<_, (Option, i64)>( + "SELECT (ai_config->>'sessions_storage_disabled')::bool, ai_sessions_backup_generation \ + FROM workspace_settings WHERE workspace_id = $1", + ) + .bind(w_id) + .fetch_optional(db) + .await? + .unwrap_or((None, 0)); + if disabled.unwrap_or(false) { + return Ok(None); + } + let (_, resource) = + crate::job_helpers_oss::get_workspace_s3_resource(authed, db, None, w_id, None).await?; + let Some(resource) = resource else { + return Ok(None); + }; + let store = build_object_store_client(&resource).await?; + let user = calculate_hash(&authed.email); + // Keyed per user, not per workspace: anyone who can write the bucket could otherwise copy + // another member's ciphertext under their own prefix and have `pull` decrypt it for them. + let key = get_workspace_key(w_id, db).await?; + let mc = crypt_from_key_with_suffix(&key, &user); + let storage_id = storage_id(&resource); + let prefix = format!("{}/{user}", generation_prefix(w_id, generation)); + Ok(Some(Backend { store, mc, prefix, storage_id, generation })) +} + +#[derive(Serialize)] +struct SessionListing { + id: String, + updated_at: chrono::DateTime, + /// The session's move count when this copy was pushed (see `PushedSession::epoch`). + epoch: u32, +} + +#[derive(Serialize)] +struct ListResponse { + enabled: bool, + /// The storage answered from, and the generation a key rotation bumps; a browser whose + /// sync state names another storage or generation starts over. + #[serde(skip_serializing_if = "Option::is_none")] + storage_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + backup_generation: Option, + sessions: Vec, + /// The user has more sessions than the answer names. + #[serde(skip_serializing_if = "std::ops::Not::not")] + truncated: bool, +} + +/// A session is listed once a push entry of it landed whole (its marker is written last); +/// a push that failed before that left objects the listing does not name. +async fn list( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, +) -> JsonResult { + require_plain_user_token(&authed)?; + let Some(backend) = backend(&authed, &db, &w_id).await? else { + return Ok(Json(ListResponse { + enabled: false, + storage_id: None, + backup_generation: None, + sessions: vec![], + truncated: false, + })); + }; + let prefix = backend.index_prefix(); + let mut stream = backend.store.list(Some(&prefix)); + // One marker per session, whatever the session holds: the newest LIST_MAX are kept as + // the scan goes (a min-heap drops the oldest), and the scan itself is bounded. + let mut newest: std::collections::BinaryHeap< + std::cmp::Reverse<(chrono::DateTime, u32, String)>, + > = Default::default(); + let mut scanned = 0; + let mut truncated = false; + while let Some(meta) = stream.next().await { + let meta = meta.map_err(object_store_error_to_error)?; + scanned += 1; + if scanned > MAX_LIST_SCAN { + truncated = true; + break; + } + // `Path` drops the trailing delimiter, so the remainder starts with one. + let Some(rel) = meta.location.as_ref().strip_prefix(prefix.as_ref()) else { + continue; + }; + let Some((sid, epoch)) = rel.trim_start_matches('/').split_once('/') else { + continue; + }; + let Ok(epoch) = epoch.parse::() else { + continue; + }; + if sid.is_empty() || sid.contains('/') { + continue; + } + newest.push(std::cmp::Reverse(( + meta.last_modified, + epoch, + sid.to_string(), + ))); + if newest.len() > LIST_MAX { + newest.pop(); + truncated = true; + } + } + let mut sessions: Vec = newest + .into_iter() + .map(|std::cmp::Reverse((updated_at, epoch, id))| SessionListing { id, updated_at, epoch }) + .collect(); + sessions.sort_by(|a, b| b.updated_at.cmp(&a.updated_at)); + Ok(Json(ListResponse { + enabled: true, + storage_id: Some(backend.storage_id.clone()), + backup_generation: Some(backend.generation), + sessions, + truncated, + })) +} + +#[derive(Deserialize)] +struct PullRequest { + ids: Vec, + /// Picks the session an earlier answer cut up from where it stopped; `ids` then names + /// that session alone. + #[serde(default)] + resume: Option, +} + +/// Where a pull of a session that outgrew one answer picks up: the last key the earlier +/// answer carried, in the session's prefix or, once that one is done, in its images prefix. +#[derive(Serialize, Deserialize, Clone)] +struct PullCursor { + id: String, + images: bool, + after: String, +} + +#[derive(Serialize)] +struct PulledChat { + id: String, + record: Box, +} + +#[derive(Serialize, Deserialize)] +struct ImageObject { + chat_id: String, + id: String, + data_url: String, +} + +#[derive(Serialize)] +struct PulledSession { + id: String, + head: Box, + chats: Vec, + images: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + artifacts: Option>, + /// The session did not fit this answer whole: the rest follows a pull with this cursor. + #[serde(skip_serializing_if = "Option::is_none")] + next: Option, + /// A fingerprint of the session's listing (marker, and every key, size, modification + /// time, entity tag and version), so the browser tells that the backup changed between + /// the pages it assembled. + listing: String, + /// The backup kept changing while this page was read (a push landing object by object), + /// so the page may mix two versions: the browser starts the session over. + #[serde(skip_serializing_if = "std::ops::Not::not")] + moved: bool, +} + +/// How many times a page whose listing moved while it was read is read again before it is +/// handed over as `moved`. +const PULL_REREADS: usize = 3; + +#[derive(Serialize)] +struct PullResponse { + enabled: bool, + #[serde(skip_serializing_if = "Option::is_none")] + storage_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + backup_generation: Option, + sessions: Vec, + /// Ids that did not fit the response budget; ask for them again. + deferred: Vec, +} + +fn raw(kind: &str, text: String) -> Result> { + RawValue::from_string(text) + .map_err(|e| Error::internal_err(format!("stored {kind} is not JSON: {e}"))) +} + +enum PullStep { + Absent, + Deferred, + Fetched(PulledSession, usize), +} + +/// Fetch one session: its head, then every chat and artifact object under its prefix, and +/// as many of its images as the budget allows (a missing image hydrates to a placeholder +/// in the browser). Sizes come from the listings, so a session that would not fit is +/// deferred before anything of it is read, unless it is the first of the response, which +/// must carry something. `Absent` when it has no head. +async fn pull_session( + backend: &Backend, + sid: &str, + budget: usize, + first: bool, + resume: Option<&PullCursor>, +) -> Result { + // A page read while a push lands object by object may mix two versions of the session: + // the listing is taken again once the page is read, and a page it moved under is read + // again, a few times, then handed over as such for the browser to start over. + for reread in 0..PULL_REREADS { + let step = pull_page(backend, sid, budget, first, resume).await?; + let PullStep::Fetched(mut page, size) = step else { + return Ok(step); + }; + if backend.listing_fingerprint(sid).await?.as_deref() == Some(page.listing.as_str()) { + return Ok(PullStep::Fetched(page, size)); + } + if reread + 1 == PULL_REREADS { + page.moved = true; + return Ok(PullStep::Fetched(page, size)); + } + } + unreachable!("a page is answered on the last reread") +} + +async fn pull_page( + backend: &Backend, + sid: &str, + budget: usize, + first: bool, + resume: Option<&PullCursor>, +) -> Result { + // Taken before anything of the page is listed or read: an object landing after it is + // in the next page's fingerprint, whereas one landing after the reads but before a + // fingerprint taken then would have certified a page without it. A session the storage + // does not list (removed, or a whole push in progress) is absent. + let Some(listing) = backend.listing_fingerprint(sid).await? else { + return Ok(PullStep::Absent); + }; + let Read::Text(head) = backend + .get(&backend.head_key(sid), MAX_HEAD_BYTES + CIPHER_PADDING) + .await? + else { + return Ok(PullStep::Absent); + }; + let session_prefix = backend.session_prefix(sid); + let images_prefix = backend.images_prefix(sid); + let mut size = head.len(); + let mut chats = vec![]; + let mut artifacts = None; + let mut images = vec![]; + let mut next = None; + let cursor = |images: bool, after: String| PullCursor { id: sid.to_string(), images, after }; + // Sizes come from the listings, and the listings stop at the budget, so nothing is read + // past it even for the first session of the answer. One that outgrew it (chats + // accumulate over pushes) comes back in pages, in key order, each answer naming where + // the next picks up; the browser imports nothing before the last page. An object that + // grew since the listing (a push replaced it) ends the page just before it, and the + // answer names that spot: a new listing sizes it, whereas dropping it would import the + // session without it for good. + let in_images = resume.is_some_and(|c| c.images); + if !in_images { + let after = resume.map(|c| ObjectPath::from(c.after.as_str())); + let (entries, cut) = backend + .list_within( + &session_prefix, + after.as_ref(), + budget.saturating_sub(size), + first, + ) + .await?; + if cut && !first { + return Ok(PullStep::Deferred); + } + if cut { + next = entries + .last() + .map(|(key, _)| cursor(false, key.to_string())); + } + let to_read: Vec<(ObjectPath, usize, Option)> = entries + .into_iter() + .filter_map(|(key, bytes)| { + let rel = key + .as_ref() + .strip_prefix(session_prefix.as_ref()) + .unwrap_or_default() + .trim_start_matches('/'); + if rel == "artifacts.json" { + Some((key, bytes, None)) + } else { + let cid = rel + .strip_prefix("chats/")? + .strip_suffix(".json")? + .to_string(); + Some((key, bytes, Some(cid))) + } + }) + .collect(); + let reads: Vec<(ObjectPath, usize, Option, Read)> = futures::stream::iter(to_read) + .map(|(key, bytes, cid)| async move { + let read = backend.get(&key, bytes).await?; + Ok::<_, Error>((key, bytes, cid, read)) + }) + .buffered(IO_CONCURRENCY) + .try_collect() + .await?; + let mut before = resume.map(|c| c.after.clone()).unwrap_or_default(); + for (key, bytes, cid, read) in reads { + match (cid, read) { + (_, Read::Grown) => { + next = Some(cursor(false, before)); + break; + } + (None, Read::Text(text)) => { + artifacts = Some(raw("artifacts", text)?); + size += bytes; + } + (Some(cid), Read::Text(text)) => { + chats.push(PulledChat { id: cid, record: raw("chat", text)? }); + size += bytes; + } + _ => {} + } + before = key.to_string(); + } + } + if next.is_none() { + let after = resume + .filter(|c| c.images) + .map(|c| ObjectPath::from(c.after.as_str())); + let (entries, cut) = backend + .list_within( + &images_prefix, + after.as_ref(), + budget.saturating_sub(size), + first, + ) + .await?; + let mut before = after.map(|a| a.to_string()).unwrap_or_default(); + if cut { + // An answer with no room for a single image names where it stood, so the pull + // owed the session alone picks it up there. + next = Some(cursor( + true, + entries + .last() + .map(|(key, _)| key.to_string()) + .unwrap_or_else(|| before.clone()), + )); + } + let to_read: Vec<(ObjectPath, usize, String, String)> = entries + .into_iter() + .filter_map(|(key, bytes)| { + let rel = key.as_ref().strip_prefix(images_prefix.as_ref())?; + let (cid, iid) = rel.trim_start_matches('/').split_once('/')?; + Some((key.clone(), bytes, cid.to_string(), iid.to_string())) + }) + .collect(); + let reads: Vec<(ObjectPath, usize, String, String, Read)> = futures::stream::iter(to_read) + .map(|(key, bytes, cid, iid)| async move { + let read = backend.get(&key, bytes).await?; + Ok::<_, Error>((key, bytes, cid, iid, read)) + }) + .buffered(IO_CONCURRENCY) + .try_collect() + .await?; + for (key, bytes, chat_id, id, read) in reads { + match read { + Read::Grown => { + next = Some(cursor(true, before)); + break; + } + Read::Text(data_url) => { + images.push(ImageObject { chat_id, id, data_url }); + size += bytes; + } + _ => {} + } + before = key.to_string(); + } + } + Ok(PullStep::Fetched( + PulledSession { + id: sid.to_string(), + head: raw("head", head)?, + chats, + images, + artifacts, + next, + listing, + moved: false, + }, + size, + )) +} + +async fn pull( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Json(req): Json, +) -> JsonResult { + require_plain_user_token(&authed)?; + if req.ids.len() > MAX_PULL_IDS { + return Err(Error::BadRequest(format!( + "at most {MAX_PULL_IDS} sessions per pull" + ))); + } + for id in &req.ids { + require_valid_id("session", id)?; + } + if let Some(cursor) = &req.resume { + if req.ids.len() != 1 || req.ids[0] != cursor.id || cursor.after.len() > 1024 { + return Err(Error::BadRequest( + "a resumed pull names the resumed session alone".to_string(), + )); + } + } + let Some(backend) = backend(&authed, &db, &w_id).await? else { + return Ok(Json(PullResponse { + enabled: false, + storage_id: None, + backup_generation: None, + sessions: vec![], + deferred: vec![], + })); + }; + let mut sessions = vec![]; + let mut deferred = vec![]; + let mut budget = PULL_RESPONSE_BUDGET; + for sid in req.ids { + let resume = req.resume.as_ref().filter(|c| c.id == sid); + match pull_session(&backend, &sid, budget, sessions.is_empty(), resume).await? { + PullStep::Absent => {} + PullStep::Deferred => deferred.push(sid), + PullStep::Fetched(session, size) => { + budget = budget.saturating_sub(size); + sessions.push(session); + } + } + } + Ok(Json(PullResponse { + enabled: true, + storage_id: Some(backend.storage_id), + backup_generation: Some(backend.generation), + sessions, + deferred, + })) +} + +#[derive(Deserialize)] +struct PushedChat { + id: String, + record: Box, +} + +#[derive(Deserialize)] +struct ImageRef { + chat_id: String, + id: String, +} + +#[derive(Deserialize)] +struct PushedSession { + id: String, + #[serde(default)] + head: Option>, + #[serde(default)] + chats: Vec, + #[serde(default)] + images: Vec, + #[serde(default)] + artifacts: Option>, + #[serde(default)] + delete_chats: Vec, + #[serde(default)] + delete_images: Vec, + /// More parts of the session follow, in this push or a later one: the session is not + /// listed on this one. + #[serde(default)] + partial: bool, + /// A part of a push of the session whole: the head is on the part that opens it, which + /// replaces whatever the storage holds of the session, and every piece the browser has + /// is on one of them. An incremental part instead rides on a session the storage lists, + /// and is refused with `needs_whole` when it lists none. + #[serde(default)] + whole: bool, + /// A push split over several parts names itself on each of them with a token the + /// browser draws; the part that `opens` it unlists the session (a pull between two parts + /// would otherwise take a mix of old and new pieces for the backup) and the last part + /// lists it again. A later part is written only while that token is the one there, so a + /// part of a push another one superseded is refused with `needs_whole`. + #[serde(default)] + push: Option, + #[serde(default)] + opens: bool, + /// The session's move count (its record's `moves`), the marker that lists the session + /// is named by: a session moved to another workspace is listed by both until the old + /// copy's removal lands, and the copy with the higher count is the later one. An + /// incremental part rides on the marker of the same count. + #[serde(default)] + epoch: u32, +} + +#[derive(Deserialize)] +struct PushRequest { + /// The email the browser believes it is acting for. An in-place account switch can + /// leave a flush prepared for the previous user; the server refuses it rather than + /// filing that user's sessions under the caller's prefix. + owner: String, + #[serde(default)] + sessions: Vec, + #[serde(default)] + removed: Vec, +} + +#[derive(Serialize)] +struct PushResult { + id: String, + #[serde(skip_serializing_if = "Option::is_none")] + error: Option, + /// Nothing was written; the session must be pushed whole again: an incremental part + /// found no listed session to ride on (another device removed the backup, or a push + /// split over parts is in progress or was abandoned), or a later part of a push split + /// over parts found another push had superseded it. + #[serde(skip_serializing_if = "std::ops::Not::not")] + needs_whole: bool, +} + +#[derive(Serialize)] +struct PushResponse { + enabled: bool, + #[serde(skip_serializing_if = "Option::is_none")] + storage_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + backup_generation: Option, + results: Vec, +} + +fn validate_push(req: &PushRequest) -> Result<()> { + if req.sessions.len() > MAX_PUSH_SESSIONS || req.removed.len() > MAX_REMOVED { + return Err(Error::BadRequest( + "too many sessions in one push".to_string(), + )); + } + // Every nested entry costs an object-store call (a deleted chat two), so the lists are + // bounded per entry and across the request; the browser sends far fewer. + let mut operations = req.removed.len(); + for s in &req.sessions { + if s.chats.len() > MAX_CHATS_PER_ENTRY + || s.images.len() > MAX_IMAGES_PER_ENTRY + || s.delete_chats.len() > MAX_DELETES_PER_ENTRY + || s.delete_images.len() > MAX_DELETES_PER_ENTRY + { + return Err(Error::BadRequest(format!( + "too many pieces for session {} in one push", + s.id + ))); + } + operations += s.chats.len() + s.images.len() + s.delete_chats.len() + s.delete_images.len(); + } + if operations > MAX_OPERATIONS_PER_PUSH { + return Err(Error::BadRequest("too many pieces in one push".to_string())); + } + for sid in &req.removed { + require_valid_id("session", sid)?; + } + for s in &req.sessions { + require_valid_id("session", &s.id)?; + if let Some(token) = &s.push { + require_valid_id("push", token)?; + } else if s.opens || s.partial { + // A part more parts follow belongs to a push split over parts, which names + // itself: without the token the session would stay listed between the parts. + return Err(Error::BadRequest(format!( + "session {} is pushed in parts with no push token", + s.id + ))); + } + if s.whole && (s.push.is_none() || s.opens) && s.head.is_none() { + return Err(Error::BadRequest(format!( + "session {} is pushed whole without its head", + s.id + ))); + } + if let Some(head) = &s.head { + require_json_object("head", head, MAX_HEAD_BYTES)?; + } + for c in &s.chats { + require_valid_id("chat", &c.id)?; + require_json_object("chat record", &c.record, PUSH_BODY_LIMIT)?; + } + if let Some(a) = &s.artifacts { + require_json_object("artifacts", a, PUSH_BODY_LIMIT)?; + } + for i in &s.images { + require_valid_id("chat", &i.chat_id)?; + require_valid_id("image", &i.id)?; + require_data_url(&i.data_url)?; + } + for c in &s.delete_chats { + require_valid_id("chat", c)?; + } + for i in &s.delete_images { + require_valid_id("chat", &i.chat_id)?; + require_valid_id("image", &i.id)?; + } + } + Ok(()) +} + +fn push_payload_bytes(req: &PushRequest) -> usize { + req.sessions + .iter() + .map(|s| { + s.head.as_ref().map_or(0, |h| h.get().len()) + + s.artifacts.as_ref().map_or(0, |a| a.get().len()) + + s.chats.iter().map(|c| c.record.get().len()).sum::() + + s.images.iter().map(|i| i.data_url.len()).sum::() + }) + .sum() +} + +/// Runs under the session's lock (see `lock_session`). The part that opens a whole push +/// (the one with the head) replaces the backup: the marker goes first, so nothing lists the +/// session until the last part, then everything else. An incremental part assumes the rest +/// of the session is in the storage, which a removal since would have taken, or a push +/// split over parts may still be bringing: it is refused unless the session is listed, and +/// unlists the session itself while it changes more than one object (a pull between two +/// writes would otherwise take a mix of old and new pieces for the backup). A push split +/// over parts names itself with a token: the part that opens it unlists the session and +/// writes the token, and a later part is written only while that token is +/// the one there, so two devices pushing the session at once cannot list a mix of their +/// pieces: the push that opened later wins, the other is refused and goes again. Every +/// refusal comes before anything of the part lands. Deletes run last, and the marker only +/// by the last part, so a push cut short never leaves a listed session pointing at chats +/// that are not there. +/// Bytes written, and whether the part was refused for the session to go whole. +async fn push_session(backend: &Backend, s: &PushedSession) -> Result<(usize, bool)> { + match &s.push { + Some(token) if !s.opens => { + match backend + .get(&backend.push_key(&s.id), token.len() + CIPHER_PADDING) + .await? + { + Read::Text(current) if current == *token => {} + _ => return Ok((0, true)), + } + } + _ => { + if s.whole { + backend + .delete_prefix(&backend.index_session_prefix(&s.id)) + .await?; + backend + .delete_prefix(&backend.session_prefix(&s.id)) + .await?; + backend.delete_prefix(&backend.images_prefix(&s.id)).await?; + } else { + if !backend.exists(&backend.index_key(&s.id, s.epoch)).await? { + return Ok((0, true)); + } + // Unlisted while more than one object changes (a push split over parts, or + // one part touching several pieces): a pull between two of the writes, or + // after one of them failed, would otherwise take a mix of old and new + // pieces for the backup. One object changing is one write. + let pieces = s.chats.len() + + s.images.len() + + usize::from(s.artifacts.is_some()) + + usize::from(s.head.is_some()) + + s.delete_chats.len() + + s.delete_images.len(); + if s.push.is_some() || pieces > 1 { + backend + .delete_prefix(&backend.index_session_prefix(&s.id)) + .await?; + } + } + if let Some(token) = &s.push { + backend + .put(&backend.push_key(&s.id), token.as_bytes()) + .await?; + } + } + } + let mut written = 0; + written += backend + .put_all( + s.images + .iter() + .map(|img| { + ( + backend.image_key(&s.id, &img.chat_id, &img.id), + backend.seal(img.data_url.as_bytes()), + ) + }) + .collect(), + ) + .await?; + written += backend + .put_all( + s.chats + .iter() + .map(|c| { + ( + backend.chat_key(&s.id, &c.id), + backend.seal(c.record.get().as_bytes()), + ) + }) + .collect(), + ) + .await?; + if let Some(a) = &s.artifacts { + written += backend + .put(&backend.artifacts_key(&s.id), a.get().as_bytes()) + .await?; + } + if let Some(h) = &s.head { + written += backend + .put(&backend.head_key(&s.id), h.get().as_bytes()) + .await?; + } + for cid in &s.delete_chats { + backend.delete(&backend.chat_key(&s.id, cid)).await?; + backend + .delete_prefix(&backend.chat_images_prefix(&s.id, cid)) + .await?; + } + backend + .delete_all( + s.delete_images + .iter() + .map(|i| backend.image_key(&s.id, &i.chat_id, &i.id)) + .collect(), + ) + .await?; + if s.partial { + return Ok((written, false)); + } + // Last, and by the last part only, so a session is listed once its whole entry landed. + backend + .store + .put(&backend.index_key(&s.id, s.epoch), PutPayload::new()) + .await + .map_err(object_store_error_to_error)?; + if s.push.is_some() { + backend.delete(&backend.push_key(&s.id)).await?; + } + Ok((written, false)) +} + +/// The marker goes first so a removal cut short leaves nothing listed, then the head so +/// nothing pulls either, and no push takes it for a session still there (see `push_session`). +async fn remove_session(backend: &Backend, sid: &str) -> Result<()> { + backend + .delete_prefix(&backend.index_session_prefix(sid)) + .await?; + backend.delete(&backend.head_key(sid)).await?; + backend.delete_prefix(&backend.session_prefix(sid)).await?; + backend.delete_prefix(&backend.images_prefix(sid)).await +} + +/// One writer per session at a time, across servers: a push and a removal of the same +/// session interleaving object by object could leave a listed session missing pieces, or a +/// marker over nothing. The lock lives in a transaction that writes no rows; it is released +/// when the transaction ends. A wait past the timeout fails that entry only, and the +/// browser retries it with backoff. +async fn lock_session( + db: &DB, + backend: &Backend, + sid: &str, +) -> Result> { + let mut tx = db.begin().await?; + sqlx::query("SET LOCAL lock_timeout = '30s'") + .execute(&mut *tx) + .await?; + sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended($1, 0::int8))") + .bind(format!("ai_session_backup:{}/{sid}", backend.prefix)) + .execute(&mut *tx) + .await + .map_err(|e| { + Error::internal_err(format!( + "another device is writing the backup of session {sid}; retried later ({e})" + )) + })?; + Ok(tx) +} + +async fn push_session_locked( + db: &DB, + backend: &Backend, + s: &PushedSession, +) -> Result<(usize, bool)> { + let tx = lock_session(db, backend, &s.id).await?; + let result = push_session(backend, s).await; + tx.commit().await?; + result +} + +async fn remove_session_locked(db: &DB, backend: &Backend, sid: &str) -> Result<()> { + let tx = lock_session(db, backend, sid).await?; + let result = remove_session(backend, sid).await; + tx.commit().await?; + result +} + +async fn push( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Json(req): Json, +) -> JsonResult { + require_plain_user_token(&authed)?; + if req.owner != authed.email { + return Err(Error::Generic( + http::StatusCode::CONFLICT, + "this push was prepared for another user".to_string(), + )); + } + validate_push(&req)?; + let Some(backend) = backend(&authed, &db, &w_id).await? else { + return Ok(Json(PushResponse { + enabled: false, + storage_id: None, + backup_generation: None, + results: vec![], + })); + }; + #[cfg(not(feature = "enterprise"))] + { + let remaining = + crate::job_helpers_oss::ce_storage_quota_remaining(&db, &w_id, None).await?; + if push_payload_bytes(&req) as i64 > remaining { + return Err(Error::QuotaExceeded( + "the workspace storage quota leaves no room for this AI session backup".to_string(), + )); + } + } + #[cfg(feature = "enterprise")] + let _ = push_payload_bytes(&req); + + let mut results = Vec::with_capacity(req.sessions.len() + req.removed.len()); + let mut written: usize = 0; + // A session split into several entries is listed by the last: once one part failed, the + // later ones are not written, or the marker would list a session missing a part. + let mut failed: std::collections::HashSet<&str> = Default::default(); + for s in &req.sessions { + let (error, needs_whole) = if failed.contains(s.id.as_str()) { + ( + Some("an earlier part of this session in the push failed".to_string()), + false, + ) + } else { + match push_session_locked(&db, &backend, s).await { + Ok((n, needs_whole)) => { + written += n; + (None, needs_whole) + } + Err(e) => { + tracing::warn!("AI session backup push failed for {} in {w_id}: {e}", s.id); + failed.insert(&s.id); + (Some(e.to_string()), false) + } + } + }; + results.push(PushResult { id: s.id.clone(), error, needs_whole }); + } + for sid in &req.removed { + let error = remove_session_locked(&db, &backend, sid) + .await + .err() + .map(|e| { + tracing::warn!("AI session backup removal failed for {sid} in {w_id}: {e}"); + e.to_string() + }); + results.push(PushResult { id: sid.clone(), error, needs_whole: false }); + } + // Overwrites and deletes make this an over-count; the periodic recount the quota check + // schedules once usage is stale settles it. + #[cfg(not(feature = "enterprise"))] + if written > 0 { + crate::job_helpers_oss::bump_storage_usage( + &db, + &w_id, + windmill_object_store::DEFAULT_STORAGE, + written as i64, + ) + .await; + } + #[cfg(feature = "enterprise")] + let _ = written; + Ok(Json(PushResponse { + enabled: true, + storage_id: Some(backend.storage_id), + backup_generation: Some(backend.generation), + results, + })) +} diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index fc90852ae6..715f0b6556 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -69,6 +69,8 @@ mod ai; #[cfg(feature = "private")] mod ai_free_tier_ee; mod ai_free_tier_oss; +#[cfg(feature = "parquet")] +mod ai_sessions; mod ai_shared_artifacts; mod apps; mod apps_raw_bundle; diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index da8532e9d1..55f244c143 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -147,6 +147,7 @@ async fn edit_copilot_config( let workspace_has_config = ai_config.has_providers(); let copilot_disabled = ai_config.copilot_disabled; + let sessions_storage_disabled = ai_config.sessions_storage_disabled; let instance_ai_config = sqlx::query_scalar!("SELECT value FROM global_settings WHERE name = 'ai_config'") .fetch_optional(&db) @@ -174,6 +175,7 @@ async fn edit_copilot_config( AIConfig::default() }; effective_ai_config.copilot_disabled = copilot_disabled; + effective_ai_config.sessions_storage_disabled = sessions_storage_disabled; Ok(Json(EditCopilotConfigResponse { effective_ai_config, @@ -212,6 +214,9 @@ async fn get_copilot_info( let copilot_disabled = workspace_ai_config .as_ref() .is_some_and(|c| c.0.copilot_disabled); + let sessions_storage_disabled = workspace_ai_config + .as_ref() + .is_some_and(|c| c.0.sessions_storage_disabled); let instance_config = sqlx::query_scalar!("SELECT value FROM global_settings WHERE name = 'ai_config'") .fetch_optional(&db) @@ -236,6 +241,7 @@ async fn get_copilot_info( AIConfig::default() }; effective.copilot_disabled = copilot_disabled; + effective.sessions_storage_disabled = sessions_storage_disabled; Ok(Json(effective)) } diff --git a/backend/windmill-common/src/variables.rs b/backend/windmill-common/src/variables.rs index 045ddcd95b..1feb4182ab 100644 --- a/backend/windmill-common/src/variables.rs +++ b/backend/windmill-common/src/variables.rs @@ -252,12 +252,17 @@ pub async fn build_crypt_with_key_suffix( key_suffix: &str, ) -> crate::error::Result { let key = get_workspace_key(w_id, db).await?; + Ok(crypt_from_key_with_suffix(&key, key_suffix)) +} + +/// The cipher `build_crypt_with_key_suffix` builds, from a key string in hand. +pub fn crypt_from_key_with_suffix(key: &str, key_suffix: &str) -> MagicCrypt256 { let crypt_key = if let Some(ref salt) = SECRET_SALT.as_ref() { format!("{}{}{}", key, salt, key_suffix) } else { format!("{}{}", key, key_suffix) }; - Ok(magic_crypt::new_magic_crypt!(crypt_key, 256)) + magic_crypt::new_magic_crypt!(crypt_key, 256) } pub async fn get_workspace_key(w_id: &str, db: &DB) -> crate::error::Result { diff --git a/backend/windmill-object-store/src/lib.rs b/backend/windmill-object-store/src/lib.rs index c403625e00..25dc6ea19c 100644 --- a/backend/windmill-object-store/src/lib.rs +++ b/backend/windmill-object-store/src/lib.rs @@ -68,8 +68,8 @@ pub mod object_store_reexports { pub use object_store::path::Path; pub use object_store::{ Attribute, Attributes, Error as ObjectStoreError, GetOptions, GetRange, GetResult, - ObjectStore, PutMultipartOpts, PutPayload, PutResult, Result as ObjectStoreResult, - WriteMultipart, + ObjectMeta, ObjectStore, PutMode, PutMultipartOpts, PutOptions, PutPayload, PutResult, + Result as ObjectStoreResult, UpdateVersion, WriteMultipart, }; } diff --git a/docs/ai-session-backups.md b/docs/ai-session-backups.md new file mode 100644 index 0000000000..0b4c550456 --- /dev/null +++ b/docs/ai-session-backups.md @@ -0,0 +1,266 @@ +# AI session backups + +AI sessions live in the browser: the session list (`windmill-sessions`), chat transcripts and +image blobs (`copilot-chat-history`) and artifacts (`copilot-artifacts`), all per-user IndexedDB +stores. This is the design of their backup in the workspace's object storage, and the +constraints future work on either side must keep. + +Backend: `backend/windmill-api/src/ai_sessions.rs` (`/w/{w}/ai/sessions/{list,pull,push}`). +Frontend: `frontend/src/lib/components/sessions/sessionMirror*.ts`. + +## Why it is lazy + +A session changes at the local write rate: a transcript write every 2 s while streaming, a +session-record write per new message on screen. An object in S3 is replaced whole and every PUT +is billed, so the backup deliberately does not follow that rate. Local writes only mark a session +dirty (`sessionMirrorSignal.ts`, import-free so the stores never depend on the backup). A flush +runs 15 s after the marks go quiet, at most 2 min after the first unflushed mark, when the tab is +hidden, and 10 s after load for marks a crash left behind. Marks are persisted in localStorage +(shared by the user's tabs) for that reason, one key per mark: a shared blob would let two tabs +marking different sessions at once rewrite each other's mark away. A dirty mark is a counter +bumped on every write; a push retires it by recording the counter it covered on the session's +sync row rather than deleting the mark, since two localStorage calls cannot compare-and-delete +and a bump landing between them would be lost; retired marks are not reclaimed (one small key per +session ever backed up), and the marks of unsent drafts and of workspaces that are off stay too, +each costing one lookup per flush. Only a session gone from the store has its mark deleted. Losing the last +seconds of a device that never comes back is accepted; a tab that closes normally keeps its marks. +A signal names the user whose store the write landed in (read off the store's scoped name), so +a write that completes after the logged-in user changed marks that user's session, for their +next load, rather than the current user's. + +A flush plans and sends one session at a time, filling requests of about 8 MB as it goes, so a +first backfill of a large history never holds more than one request's worth of records and +images in memory. + +## What a push carries + +The pure planner (`sessionMirrorPlan.ts`) compares each piece against the marker of what was +last pushed, kept per session in the `windmill-sessions-mirror` store: + +| Piece | Object | Sent when | +|---|---|---| +| session record | `sessions/{sid}/head.json` | its signature changed | +| chat | `sessions/{sid}/chats/{cid}.json` | its `lastModified` moved | +| artifacts | `sessions/{sid}/artifacts.json` | their fingerprint changed | +| image | `images/{sid}/{cid}/{iid}` | never pushed before (write-once) | +| index marker | `index/{sid}/{epoch}` | last, by the part that completes a push of the session (empty; named by the record's move count) | + +All under `windmill_ai_sessions/{w_id}/g{generation}/{sha256(email)}/` in the workspace's +primary storage (the generation is what a key rotation moves, see below). +The listing reads only `index/`: one object per session whatever the session holds, so a +session with many chats cannot crowd newer ones out of a bounded scan, and its +`last_modified` is the session's `updated_at`. Written last, and only by an entry no unsent +part follows (a session split over several entries says `partial` on all but the last, and +names the push on each, or the part is refused), it +lists a session only once a whole push landed; the parts of a session after a failed one are +not written either, on the server within one push and on the client across pushes, so the +marker on the last part never lists a session missing a chat, and a new session whose last part +never lands is not listed at all. A push of the session whole (no sync row, or a stale one, +`whole` on every part) opens with the head on the first: that part replaces the backup (the +marker goes first, then everything under the session), so what an old storage still held of +the session and the push does not carry is gone. An incremental part rides on a listed +session, and the server refuses it with `needs_whole`, writing nothing, when none is listed +(a removal deletes the marker first), rather than write a marker over a session missing what +earlier parts or earlier pushes carried; one that changes more than one object unlists the +session before its writes and lists it again after them, so a pull between two of the writes, +or after one failed, finds it absent rather than a mix of old and new pieces (one object +changing is one write, and stays listed). A push split over several parts, whole or +incremental, names itself on each with a token the browser draws (`push`, `opens` on the +first): the opening part unlists the session, so a pull between two parts finds it absent +rather than a mix of old and new pieces, the last part lists it again, and a later part is +written only while that token is the one there, so two devices pushing the session at once +cannot list a mix of their pieces (the push that opened later wins; the other is refused with +`needs_whole` and goes again), and one abandoned leaves the session unlisted, so the next +push of it goes whole. A push and a removal of one session +are serialized on the server by a Postgres advisory lock keyed on the session's prefix, so +the two never interleave object by object. + +The head signature leaves out `name` (a per-browser counter the sessions page routes by), +the unsent-draft fields, `workspace_root_id` (recomputed on import), and the two fields reading +a session bumps (`lastSeenCount`, `lastActivityAt`). Reading a session must never cost a push; +keep that property when adding fields to `Session`. + +Unsent drafts (no `workspace_id`) and attached files (Blobs, directory handles) are not backed up. + +## Encryption and access + +Every object is encrypted with a key derived from the workspace key and the user +(`build_crypt_with_key_suffix` with the email hash), because workspace storage credentials are +shared far more widely than a user's transcripts: `public_resource` storages and legacy-mode +READ/WRITE hand any member the bucket. The key is per user rather than per workspace so that a +member who copies another user's ciphertext under their own prefix gets nothing from `pull`; an +object that does not decrypt for its reader is treated as absent. Rotating the workspace key +(`set_encryption_key`) does not re-key the backups the way it re-encrypts the workspace's +secrets. The objects live under a prefix named by a generation +(`workspace_settings.ai_sessions_backup_generation`) that the rotation bumps in the +transaction committing the new key; once committed, the routes read and write under the new +generation's prefix, the answers name it (`backup_generation`, below; `storage_id` names the +storage and does not change), so every browser marks its sync rows stale and pushes its +sessions whole again there, and every older generation, which nothing writes to any more, is deleted off the +request at leisure (`windmill-api-workspaces/src/ai_session_backups.rs`). Sessions no browser +holds any more are lost. A generation is never reused, so no deletion, however late, can +touch live objects; a rotation that fails before its commit bumps nothing and deletes +nothing; two rotations racing serialize on the key row; the same key set again bumps +nothing. A rotation is rare, and the alternative, +rewriting every object in place while pushes, restarts, storage switches and further +rotations race the rewrite, is where the complexity would be; with this, nothing but the +current key ever reads an object. The +server builds every key from ids it validated +(`[A-Za-z0-9_-]{1,64}`) and the caller's own email; the client never names a key, and the +workspace storage permission rules are not consulted (the same stance as volumes). Only an +unscoped user token may reach the routes: a job token can carry an `on_behalf_of` identity and +every scoped token (guest, embed, app policy, MCP) was minted for something narrower. + +The backup is keyed by the email like the browser's own stores are (`userScopedDb` scopes +IndexedDB by it): a user whose email changes starts from an empty history on both sides, and +the objects under the old hash stay in the bucket unread. Carrying them over would need a +server-side re-key (decrypt with the old suffix, encrypt with the new, move every object) in the +email-change flow, which this design leaves out. + +An image is accepted only as a base64 data URL of at most 4 MB and stored verbatim, so it +serializes back into a pull answer at its stored size; anything JSON would escape could grow +several times and defeat the pull budget. + +`push` carries `owner`, the email the browser prepared the batch for, and the server refuses a +mismatch with 409: an in-place account switch must not file one user's sessions under another's +prefix. The client captures its user at flush start and checks every store handle's name +against it for the same reason. + +The feature is on wherever the workspace has primary storage, and off with +`ai_config.sessions_storage_disabled` (the `copilot_disabled` pattern: no migration, carried by +settings export and the CLI). A build without `parquet` has no routes (404), a workspace without +storage answers `enabled: false`; either turns the backup off for ten minutes, after which the +page asks again on its own (a flush for whatever is pending, and a restore), and the AI +settings page tells the mirror at once when the switch is saved there (the off state is +forgotten, the rows that went stale are marked again, a restore runs). + +## Conflicts and deletion + +Last write wins across devices. The head carries no manifest; `pull` lists the session's prefix +instead, so a stale device that renames or archives a session rewrites only the head and cannot +hide chats a newer device wrote. Two devices continuing the same chat still collide. + +Restore brings back only sessions the browser does not have (`importSessions` is write-if-absent, +and skips ids the user deleted in this page) and never overwrites or deletes a local one from +remote state. It covers the workspace and its forks together, and only once every one of them +that keeps backups has listed (a listing that failed leaves the family for the next page load +or workspace switch, or the copy that did list could be the stale one): a session listed by +two of them (moved between them, the old copy not yet removed, since that mark is the moving +browser's, which may never come back) is brought back from the copy that moved last (`epoch`, +the record's move count, which names the marker), the storage's own modification time +deciding between two of the same count, and not from the other, which would otherwise take +the id first and keep the later copy out for good. A workspace's records land together once +its pulls are done, and just before they do the whole family is listed again (members whose +backups were off included, since a move from another device can land in a workspace between +the first listings and the pulls; a family of one, with nowhere else for a copy to show up, is +not): a session a later copy of which showed up elsewhere is left, with the family, for the +next time. Only a user-initiated `deleteSession` removes the backup; the next push from +another device that still has the session is refused with `needs_whole` (nothing of it is +written), its row goes stale without a backoff, and that device's next flush sends the session +whole; the workspace-lifecycle +removals (`reconcileSessionsLifecycle`, `deleteSessionsForWorkspace`) leave it, so a session +dropped by a wrong reconcile comes back on the next restore. Objects of deleted workspaces stay +in the bucket. A session moved to another workspace is pushed whole into the new one, and once +that push has landed the copy in the old one gets a removal mark of its own, naming the storages holding +that copy (the row that knew is the new workspace's by then), retried independently until +each of them has answered, even when the old workspace's backups are off at the time (they +may hold the copy still). Filing the removal only after the new copy is acknowledged keeps the +session backed up somewhere at every point. + +A restore writes a session's artifacts and chats before its record, and records nothing for a +session whose pieces could not be written: recording it would let the next flush push the +half-empty local state over the backup. + +Every answer names the storage it came from (`storage_id`, a hash of what locates the objects, +endpoint, region and bucket, not the credentials, which rotate) and the backup generation a +key rotation bumps (`backup_generation`). A sync row records both, and a row naming another +storage or generation goes stale and its session is marked again: a workspace pointed at a +new bucket, or whose key was rotated, holds nothing, and the server looks nowhere else, so +the next flush carries the session whole. A switch leaves the old copy where it was, so the +row rewritten under the new storage records the old one (`alsoIn`, one entry per storage +the workspace was on), and a removal is done only once every storage holding a copy answered +it, whatever the generation (a rotation deleted the older generation's copy anyway): each +answer narrows the row to the storages still holding one, and the mark waits for them to +answer, so a switch back never brings a deleted session back. That includes the rows a flush has just written, when a later answer of the +same flush names another storage or the session was pushed in part on top of a row from the +old one; a session whose own parts were answered from different storages is not settled at +all. The listing a restore starts with runs the same check, so a storage switch is noticed at +the first push after it or on the next page load, whichever comes first. + +## Limits + +Push bodies are packed to about 8 MB (UTF-8 bytes as sent), at most 100 entries, 200 removals and +4000 pieces each (the server's caps, with 32 MB on the body, and 100 chats, 500 images or 1000 +deletes per entry, since every piece is an object-store call); an entry that +outgrows the target is split into chat-only parts (the artifacts and deletes on the last, the +head on the last too for an incremental push and on the first part, whatever it carries, for a +push of the session whole), and deletes +past the per-entry cap are carried over to the next push, which the session stays marked for. A chat above +16 MB or a session's artifacts above 8 MB are left out with a console warning; a chat that grew +past the cap after it was backed up has its copy deleted, so a restore never presents the old +transcript as the current one. A 413 fails only the sessions of that request. A +request the server refuses (any other 4xx but 404/403/409) stops the backup for the page but keeps +the marks and the sync state, so the next load tries again; a session the server reports it could +not store stays marked and is retried with backoff. A move files the old workspace's removal +mark before recording the new copy's row, so a mark that could not be written leaves the move +to be planned again. A workspace that answers `enabled: false` +marks its sync rows stale (the next push after storage returns carries every session whole, +since a new storage may be a new bucket) and leaves its dirty marks where they are (a move into +it must still remember the old copy); it keeps the removal marks of sessions that had been backed +up, so one deleted while backups are off does not come back once they are on, and drops the +removals of sessions never backed up from this browser, so a storage-less instance does not +collect one mark per deleted session forever. A user delete whose removal mark cannot be written +(localStorage full) is carried by the session's sync row instead (`removed`), which the flush +and the restore read like a mark; a session without a row yet (its first push may be in +flight) gets a row saying only that, and every row write keeps a removal filed meanwhile, so +the push's own row cannot erase it. Pull bodies are +capped at 64 KB. Pull answers up to 20 ids within a 32 MB +budget: a session's size is known from the listings before anything of it is read, one that +would not fit is deferred unless it is the first of the answer, in which case it comes in +pages: the answer carries what fits in key order (at least one object, so every page makes +progress) and names where the next picks up (`next`, a cursor the browser sends back as +`resume` with that session alone). Every page carries a fingerprint of the session's listing +(marker, keys, sizes, modification times, entity tags and versions, since a store reports +modification times coarsely and an object rewritten at the same size within that grain would +otherwise fingerprint the same) taken before anything of it is read, and the server +takes it again once the page is read: a page the backup moved under (a push landing object by +object) is read again, a few times, then answered as `moved`, and the browser starts the +session over on that or on two pages whose fingerprints differ. The browser writes each page's pieces as it arrives, over whatever +an earlier restore cut short had staged (the session is absent locally, so its pieces have no +local edits to keep, and the backup may have moved on), and the record, which is what makes +the session visible, only with the last page. A restore in progress keeps a staging row for +the session (the ids of every chat, image, artifact and version it wrote), which outlives it +if it is cut short; the next restore deletes the staged pieces the backup no longer has, by id +and never by clock, before the record lands (once the record is there no restore looks at the +session again, and a flush would push them back), and a prune that could not run leaves the +session, its pieces and its staging row for the restore after. A restore holds the user's tab +lock while it runs, so two tabs cannot each write the same absent session's pieces over the +other's, and runs only where Web Locks exist (a secure context: https, or localhost); on a plain +http origin the browser still backs up, and its sessions come back on a secure one. A +restore never writes an older record over a newer +one; between pages it holds nothing but the sync +row being assembled, whose chats also admit the images of a later page. Every page carries a +fingerprint of the session's listing (`listing`), taken before anything of the page is +listed or read, so an object landing after it is in the next page's; a session whose +fingerprint moved between two of its pages (a chat added by another device could sort before +the cursor and be missed) starts over, up to three times, then waits for the next restore. An object that grew +since the listing (a push replaced it) ends its page just before it and the answer names that +spot, so the next page sizes it anew rather than the session being imported without it. A pull sees every key of a session's listing but keeps the 5000 smallest +past its cursor (a page is defined by key order, and the store promises none), so a session +grown without bound by valid pushes cannot grow the answer's memory through its metadata +either; removing a prefix and a rotation's deletion stream their listings. `list` scans at most 50 000 index markers, keeps the newest 500 as it goes and answers with +them (`truncated` says when there were more); the restore takes 50 of them. Every read checks the object's size before buffering it: one larger than any push writes +(32 MB) is planted, whatever its listing said, and skipped, since whoever holds the bucket's +credentials can put anything at a predictable key; one larger than its listing said grew +since (a push replaced it) and ends its page, for the next pull to size anew. A dirty mark that cannot be written +(localStorage full) records its bump on the session's sync row instead (`extraV`, counted +with the mark's counter and kept by every row write, so a push in flight cannot retire it); +a session without a row yet (its first push in flight) keeps the bump in the page, and the +next row write takes it onto the row in the same transaction, so the row the push writes +cannot retire the mark with the bump unseen; every load's backfill marks again a session +without a row, with a stale one, or with one carrying bumps no push has covered. +A mark or removal for another user (a write that landed after a switch) reaches that user's +rows through a connection of its own, since the shared handle follows the current user. Nothing is read past the budget, whatever a session holds. A +restore takes the newest 50 sessions per workspace: every visible session gets a runtime, and +each runtime's history load reads the whole chat store. On CE the push checks the storage quota +and bumps usage by bytes written (an over-count on overwrites; the periodic recount settles it). diff --git a/frontend/src/lib/components/InstanceSettings.svelte b/frontend/src/lib/components/InstanceSettings.svelte index b1217b4328..f51e459f77 100644 --- a/frontend/src/lib/components/InstanceSettings.svelte +++ b/frontend/src/lib/components/InstanceSettings.svelte @@ -1084,11 +1084,12 @@ it with none, whether SSO logins evaluate an IdP groups claim (SAML or OIDC) and change a membership, the plan tier and quota shown when the execution meter is opened, whether app sandbox isolation is turned on, whether a step's workspace script is - edited from the flow editor, which skin approval steps are given, how data tables and - their migrations are set up and used, how often an empty workspace home is seen, how - often the home page’s create menu and hub-project picker are opened and from which - entry point, the name of any public hub project imported from the home page and how - far that import got, and whether a pre-approved trial offer was opened, last 30 days)
  • feature adoption (counts of which flow, script, trigger, worker and data table @@ -1149,11 +1150,12 @@ it with none, whether SSO logins evaluate an IdP groups claim (SAML or OIDC) and change a membership, the plan tier and quota shown when the execution meter is opened, whether app sandbox isolation is turned on, whether a step's workspace script is - edited from the flow editor, which skin approval steps are given, how data tables and - their migrations are set up and used, how often an empty workspace home is seen, how - often the home page’s create menu and hub-project picker are opened and from which - entry point, the name of any public hub project imported from the home page and how - far that import got, and whether a pre-approved trial offer was opened, last 30 days)
  • feature adoption (counts of which flow, script, trigger, worker and data table diff --git a/frontend/src/lib/components/copilot/chat/HistoryManager.svelte.ts b/frontend/src/lib/components/copilot/chat/HistoryManager.svelte.ts index 592b6217c8..4edac6dc0a 100644 --- a/frontend/src/lib/components/copilot/chat/HistoryManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/HistoryManager.svelte.ts @@ -1,9 +1,15 @@ -import { type DBSchema as IDBSchema, type IDBPDatabase } from 'idb' +import { + type DBSchema as IDBSchema, + type IDBPDatabase, + type IDBPTransaction, + type StoreNames +} from 'idb' import type { ChatJob, DisplayMessage } from './shared' import { expanded, messageDraft } from './chatDraft' import { createLongHash } from '$lib/editorLangUtils' import { userScopedDb, type UserScopedDbMigrateDeps } from '$lib/userScopedDb' -import { scopedKey } from '$lib/userScopedStorage' +import { emailOfScopedKey, scopedKey, scopedKeyFor } from '$lib/userScopedStorage' +import { markSessionDirty } from '$lib/components/sessions/sessionMirrorSignal' import type { ChatCompletionMessageParam } from 'openai/resources/index.mjs' import type { PersistedContextUsage } from './tokenUsage' import { IMAGE_OMITTED_PLACEHOLDER, type AttachedImage } from './imageUtils' @@ -14,8 +20,9 @@ import { randomUUID } from '$lib/utils/uuid' // shared browser. The bare name is also the legacy (pre-namespacing) DB, claimed // once on first login. const DB_NAME = 'copilot-chat-history' -// v3 adds the images blob store (replacing v2's short-lived toolImages store). -const DB_VERSION = 3 +// v3 adds the images blob store (replacing v2's short-lived toolImages store); v4 indexes +// chats by session. +const DB_VERSION = 4 /** Newest image blobs kept per chat; each is a bounded (≤1568px) data URL. */ const MAX_IMAGES_PER_CHAT = 30 /** Marks a persisted image whose bytes live in the `images` store. */ @@ -45,6 +52,7 @@ interface ChatSchema extends IDBSchema { // chats predating this feature. Persisted out-of-band like modifiedItems. backgroundJobs?: ChatJob[] } + indexes: { 'by-session': string } } // Image bytes, out-of-band from the chat record on purpose: the record is // re-cloned into IndexedDB on every saveChat, while a blob is written once @@ -64,9 +72,19 @@ interface ChatSchema extends IDBSchema { } } -function createChatStore(db: IDBPDatabase): void { - if (!db.objectStoreNames.contains('chats')) { - db.createObjectStore('chats', { keyPath: 'id' }) +/** A persisted chat, exactly as the store holds it (image refs, not bytes). */ +export type StoredChat = ChatSchema['chats']['value'] + +function createChatStore( + db: IDBPDatabase, + tx: IDBPTransaction[], 'versionchange'> +): void { + const chats = db.objectStoreNames.contains('chats') + ? tx.objectStore('chats') + : db.createObjectStore('chats', { keyPath: 'id' }) + // Lets the session backup find a session's chats without reading every record. + if (!chats.indexNames.contains('by-session')) { + chats.createIndex('by-session', 'sessionId') } // v2 briefly kept full-resolution tool screenshots in their own store; the // general blob store below covers them now. @@ -117,7 +135,9 @@ async function claimLegacyChatDb( { openDB, deleteDB }: UserScopedDbMigrateDeps ): Promise { if ((await scopedDb.count('chats')) > 0) return - const legacy = await openDB(DB_NAME, 1, { upgrade: createChatStore }) + const legacy = await openDB(DB_NAME, 1, { + upgrade: (db, _oldVersion, _newVersion, tx) => createChatStore(db, tx) + }) const legacyChats = await legacy.getAll('chats') if (legacyChats.length > 0) { const tx = scopedDb.transaction('chats', 'readwrite') @@ -156,6 +176,129 @@ export async function readChatModifiedItems(chatId: string): Promise(DB_NAME, { + version: DB_VERSION, + upgrade: createChatStore, + migrate: migrateLegacyChatDb +}) + +async function backupDb(email: string): Promise | undefined> { + const db = await backupDbh.whenReady() + return db && db.name === scopedKeyFor(DB_NAME, email) ? db : undefined +} + +/** Test-only: let go of the backup's handle so the next call opens the test's fresh + * IndexedDB rather than the connection a previous test left. */ +export function __resetBackupStoreForTesting(): void { + backupDbh.close() +} + +/** Ids of the chats tagged with this session, or undefined when the store is unavailable. */ +export async function listSessionChatIds( + sessionId: string, + email: string +): Promise { + const db = await backupDb(email) + if (!db) return undefined + return (await db.getAllKeysFromIndex('chats', 'by-session', sessionId)).map(String) +} + +export async function readStoredChat(id: string, email: string): Promise { + const db = await backupDb(email) + return db?.get('chats', id) +} + +/** Ids of the image blobs a chat owns, or undefined when the store is unavailable. */ +export async function listChatImageIds( + chatId: string, + email: string +): Promise { + const db = await backupDb(email) + if (!db) return undefined + return (await imageKeysForChat(db, chatId)).map(String) +} + +export async function readImageDataUrl(id: string, email: string): Promise { + const db = await backupDb(email) + return (await db?.get('images', id))?.dataUrl +} + +export interface RestoredImage { + id: string + chatId: string + dataUrl: string +} + +/** + * Write restored chats and image blobs, leaving any that already exist alone: a record + * this browser wrote since is newer than the backup it came from. False when the store + * could not be reached, which the caller must not record as a restore. + */ +export async function importStoredChats( + chats: StoredChat[], + images: RestoredImage[], + email: string, + overwrite = false +): Promise { + const db = await backupDb(email) + if (!db) return false + const tx = db.transaction(['chats', 'images'], 'readwrite') + const chatStore = tx.objectStore('chats') + const imageStore = tx.objectStore('images') + const savedAt = Date.now() + for (const image of images) { + if (overwrite || (await imageStore.getKey(image.id)) === undefined) { + await imageStore.put({ id: image.id, chatId: image.chatId, dataUrl: image.dataUrl, savedAt }) + } + } + // An overwrite never puts an older record over a newer one: without a cross-tab lock, + // another restore may have landed a newer backup's copy meanwhile. + for (const chat of chats) { + const existing = await chatStore.get(chat.id) + if (existing === undefined || (overwrite && existing.lastModified <= chat.lastModified)) { + await chatStore.put(chat) + } + } + await tx.done + return true +} + +/** Deletes these chats of the session (with their images) and these images: what an earlier + * restore staged for it and the backup no longer has. False when nothing could be deleted. */ +export async function pruneSessionChats( + sessionId: string, + chats: Set, + images: Set, + email: string +): Promise { + if (chats.size === 0 && images.size === 0) return true + const db = await backupDb(email) + if (!db) return false + try { + const tx = db.transaction(['chats', 'images'], 'readwrite') + const chatStore = tx.objectStore('chats') + const imageStore = tx.objectStore('images') + for (const chatId of await chatStore.index('by-session').getAllKeys(sessionId)) { + if (!chats.has(String(chatId))) continue + await chatStore.delete(chatId) + const keys = await imageStore + .index('by-chat') + .getAllKeys(IDBKeyRange.bound([chatId, -Infinity], [chatId, Infinity])) + for (const key of keys) await imageStore.delete(key) + } + for (const id of images) await imageStore.delete(id) + await tx.done + return true + } catch (err) { + console.error('Could not prune chats for session', err) + return false + } +} + export default class HistoryManager { // Per-instance handle to the shared per-user DB lifecycle. There is one // HistoryManager per AIChatManager (the singleton + one per session runtime), @@ -289,7 +432,10 @@ export default class HistoryManager { const snapshot = $state.snapshot(existing) const updated = { ...snapshot, sessionId } this.savedChats = { ...this.savedChats, [chatId]: updated } - await this.enqueueDbWrite((db) => db.put('chats', updated)) + await this.enqueueDbWrite(async (db) => { + await db.put('chats', updated) + markSessionDirty(sessionId, chatId, emailOfScopedKey(DB_NAME, db.name)) + }) } getPastChats() { @@ -565,6 +711,13 @@ export default class HistoryManager { const keep = this.keptImageIds(refs) await this.writeKeptImageBlobs(db, updatedChat.id, blobs, keep) await db.put('chats', updatedChat) + if (updatedChat.sessionId) { + markSessionDirty( + updatedChat.sessionId, + updatedChat.id, + emailOfScopedKey(DB_NAME, db.name) + ) + } // Best-effort: the record is already committed, so a failed cleanup // (e.g. a user switch closed this handle mid-op) must not turn a // successful save into a rejection — the orphans are reclaimed by @@ -589,6 +742,7 @@ export default class HistoryManager { } deletePastChat(id: string) { + const sessionId = this.savedChats[id]?.sessionId this.savedChats = Object.fromEntries( Object.entries(this.savedChats).filter(([key]) => key !== id) ) @@ -596,6 +750,7 @@ export default class HistoryManager { await db.delete('chats', id) const keys = await imageKeysForChat(db, id) await Promise.all(keys.map((key) => db.delete('images', key))) + if (sessionId) markSessionDirty(sessionId, id, emailOfScopedKey(DB_NAME, db.name)) }).catch((err) => console.error('Could not delete chat', err)) } diff --git a/frontend/src/lib/components/copilot/chat/artifacts/artifactsDB.ts b/frontend/src/lib/components/copilot/chat/artifacts/artifactsDB.ts index 94afac404e..e7ff4b1f60 100644 --- a/frontend/src/lib/components/copilot/chat/artifacts/artifactsDB.ts +++ b/frontend/src/lib/components/copilot/chat/artifacts/artifactsDB.ts @@ -2,6 +2,8 @@ // active chat's rotation, so chatId-keying would drop artifacts on each new conversation. import { type DBSchema as IDBSchema, type IDBPObjectStore, type IDBPTransaction } from 'idb' import { userScopedDb } from '$lib/userScopedDb' +import { emailOfScopedKey, scopedKeyFor } from '$lib/userScopedStorage' +import { markSessionDirty } from '$lib/components/sessions/sessionMirrorSignal' export type ArtifactKind = 'md' | 'html' @@ -89,9 +91,11 @@ interface ArtifactsSchema extends IDBSchema { } } +const ARTIFACTS_DB = 'copilot-artifacts' + // User-scoped like the chat-history store these are keyed against: no cross-user // co-residency on a shared browser. -const dbh = userScopedDb('copilot-artifacts', { +const dbh = userScopedDb(ARTIFACTS_DB, { version: 2, // Runs for a fresh database and for the v1 upgrade alike, so create each store only // when it is missing. @@ -118,11 +122,69 @@ export async function putArtifact(artifact: PersistedArtifact): Promise { // A rejected write (most likely QuotaExceededError) leaves the artifact usable for the // session but unpersisted — degrade like the reads rather than throwing at the caller. await db.put('items', artifact) + markSessionDirty(artifact.sessionId, undefined, emailOfScopedKey(ARTIFACTS_DB, db.name)) } catch (err) { console.error('Could not persist artifact', err) } } +/** A session's artifacts with their history, or undefined when the store is unavailable + * or no longer the named user's (see `readStoredSessions`). */ +export async function readSessionArtifacts( + sessionId: string, + email: string +): Promise<{ items: PersistedArtifact[]; versions: ArtifactVersion[] } | undefined> { + const db = await getDB() + if (!db || db.name !== scopedKeyFor(ARTIFACTS_DB, email)) return undefined + try { + const items = await db.getAllFromIndex('items', 'by-session', sessionId) + const versions = ( + await Promise.all(items.map((i) => db.getAllFromIndex('versions', 'by-artifact', i.id))) + ).flat() + return { items, versions } + } catch (err) { + console.error('Could not read artifacts', err) + return undefined + } +} + +/** Write restored artifacts and snapshots, leaving any that already exist alone. Reports + * whether they are all in the store now: unlike the other writes here, a caller records the + * restore as done on the strength of this answer. */ +export async function importArtifacts( + items: PersistedArtifact[], + versions: ArtifactVersion[], + email: string, + overwrite = false +): Promise { + if (items.length === 0 && versions.length === 0) return true + const db = await getDB() + if (!db || db.name !== scopedKeyFor(ARTIFACTS_DB, email)) return false + try { + const tx = db.transaction(['items', 'versions'], 'readwrite') + const itemStore = tx.objectStore('items') + const versionStore = tx.objectStore('versions') + // An overwrite never puts an older record over a newer one: without a cross-tab lock, + // another restore may have landed a newer backup's copy meanwhile. + for (const item of items) { + const existing = await itemStore.get(item.id) + if (existing === undefined || (overwrite && existing.updatedAt <= item.updatedAt)) { + await itemStore.put(item) + } + } + for (const version of versions) { + if (overwrite || (await versionStore.getKey(version.key)) === undefined) { + await versionStore.put(version) + } + } + await tx.done + return true + } catch (err) { + console.error('Could not import artifacts', err) + return false + } +} + export async function getArtifact(id: string): Promise { const db = await getDB() if (!db) return undefined @@ -273,7 +335,11 @@ export async function mutateArtifact( reportFailure = false abort() } - return { outcome: await settled, artifact: edit.artifact } + const outcome = await settled + if (outcome === 'saved' && db) { + markSessionDirty(edit.artifact.sessionId, undefined, emailOfScopedKey(ARTIFACTS_DB, db.name)) + } + return { outcome, artifact: edit.artifact } } /** @@ -342,14 +408,47 @@ export async function deleteArtifact(id: string): Promise { if (!db) return try { const tx = db.transaction(['items', 'versions'], 'readwrite') - await tx.objectStore('items').delete(id) + const items = tx.objectStore('items') + const sessionId = (await items.get(id))?.sessionId + await items.delete(id) await deleteVersionsIn(tx.objectStore('versions'), id) await tx.done + if (sessionId) markSessionDirty(sessionId, undefined, emailOfScopedKey(ARTIFACTS_DB, db.name)) } catch (err) { console.error('Could not delete artifact', err) } } +/** Deletes these artifacts of the session (with their versions) and these versions: what + * an earlier restore staged for it and the backup no longer has. False when nothing could + * be deleted. */ +export async function pruneSessionArtifacts( + sessionId: string, + itemIds: Set, + versionKeys: Set, + email: string +): Promise { + if (itemIds.size === 0 && versionKeys.size === 0) return true + const db = await getDB() + if (!db || db.name !== scopedKeyFor(ARTIFACTS_DB, email)) return false + try { + const tx = db.transaction(['items', 'versions'], 'readwrite') + const items = tx.objectStore('items') + const versions = tx.objectStore('versions') + for (const id of await items.index('by-session').getAllKeys(sessionId)) { + if (!itemIds.has(String(id))) continue + await items.delete(id) + await deleteVersionsIn(versions, String(id)) + } + for (const key of versionKeys) await versions.delete(key) + await tx.done + return true + } catch (err) { + console.error('Could not prune artifacts for session', err) + return false + } +} + export async function deleteArtifactsForSession(sessionId: string): Promise { const db = await getDB() if (!db) return diff --git a/frontend/src/lib/components/sessions/sessionMirror.svelte.ts b/frontend/src/lib/components/sessions/sessionMirror.svelte.ts new file mode 100644 index 0000000000..b1ac4bcd06 --- /dev/null +++ b/frontend/src/lib/components/sessions/sessionMirror.svelte.ts @@ -0,0 +1,1644 @@ +// Lazily backs the browser's AI sessions up to their workspace's object storage, and +// restores the ones this browser does not have. +// +// IndexedDB stays the store every write lands in; the funnels there only mark a session +// dirty (sessionMirrorSignal). A flush runs once the marks have been quiet for a while, +// bounded by a maximum delay so a long turn still gets backed up part-way, and sends +// batched requests per workspace carrying only the pieces whose marker moved +// (sessionMirrorPlan). Marks are persisted, so a crash leaves them for the next load. +// +// What a flush is for is decided per workspace: `enabled: false` (no storage, or the +// admin switch) turns it off for ten minutes, after which the page asks again on its own, +// and at once when the switch is saved from this page (`backupSettingsChanged`). +import { BROWSER } from 'esm-env' +import { get } from 'svelte/store' +import { openDB, type DBSchema, type IDBPDatabase } from 'idb' +import { + AiService, + ApiError, + type AISessionBackup, + type AISessionBackupCursor, + type AISessionBackupImage, + type AISessionBackupPush +} from '$lib/gen' +import { userWorkspaces } from '$lib/stores' +import { userScopedDb } from '$lib/userScopedDb' +import { getCurrentUserEmail, onUserChange, scopedKey, scopedKeyFor } from '$lib/userScopedStorage' +import { logFeatureUsage } from '$lib/utils/featureUsage' +import { randomUUID } from '$lib/utils/uuid' +import { workspaceRootId } from './sessionScope.svelte' +import { onMirrorSignal } from './sessionMirrorSignal' +import { + importSessions, + isSessionTombstoned, + readStoredSessions, + sessionState, + type Session +} from './sessionState.svelte' +import { + importStoredChats, + pruneSessionChats, + listChatImageIds, + listSessionChatIds, + readImageDataUrl, + readStoredChat, + type RestoredImage, + type StoredChat +} from '../copilot/chat/HistoryManager.svelte' +import { + importArtifacts, + pruneSessionArtifacts, + readSessionArtifacts, + type ArtifactVersion, + type PersistedArtifact +} from '../copilot/chat/artifacts/artifactsDB' +import { + artifactsFingerprint, + headSig, + jsonBytes, + operationsOf, + planSessionPush, + splitEntry, + type ChatSnapshot, + type MirrorSyncState, + type PlannedPush, + type PushBody +} from './sessionMirrorPlan' + +/** A flush waits for the marks to go quiet this long. */ +const QUIET_MS = 15_000 +/** ...but never longer than this after the first unflushed mark. */ +const MAX_DELAY_MS = 120_000 +const STARTUP_DELAY_MS = 10_000 +const RETRY_MIN_MS = 30_000 +const RETRY_MAX_MS = 600_000 +/** Requests are packed up to about this many bytes; the server accepts four times that. */ +const REQUEST_TARGET_BYTES = 8 * 1024 * 1024 +/** The server's caps per request and per entry. */ +const MAX_ENTRIES_PER_REQUEST = 100 +const MAX_REMOVED_PER_REQUEST = 200 +const MAX_IMAGES_PER_ENTRY = 500 +const MAX_OPERATIONS_PER_REQUEST = 4000 +/** A chat or a session's artifacts beyond this are left out of the backup rather than + * sent: with the record and the deletes riding along, the largest entry stays well under + * the server's 32 MB body cap. */ +const MAX_CHAT_BYTES = 16 * 1024 * 1024 +const MAX_ARTIFACTS_BYTES = 8 * 1024 * 1024 +const PULL_BATCH = 5 +/** Newest sessions restored per workspace: every visible session gets a runtime, and each + * runtime's history load reads the whole chat store. */ +const RESTORE_MAX = 50 +const PENDING_PREFIX = 'windmill_sessions_mirror_pending' +const SYNC_DB = 'windmill-sessions-mirror' + +interface MirrorSchema extends DBSchema { + sync: { key: string; value: MirrorSyncState } +} + +function createSyncStore(db: IDBPDatabase): void { + if (!db.objectStoreNames.contains('sync')) db.createObjectStore('sync', { keyPath: 'id' }) +} + +const syncDbh = userScopedDb(SYNC_DB, { version: 1, upgrade: createSyncStore }) + +async function syncDb(email: string) { + const db = await syncDbh.whenReady() + return db && db.name === scopedKeyFor(SYNC_DB, email) ? db : undefined +} + +// --- Pending marks --- +// +// One localStorage key per mark, shared by every tab of the user: a dirty mark holds a +// counter bumped on every write, a removal mark the workspace to remove from and, for a +// session that moved to another workspace, the storages holding the old copy (the row that +// knew is the new workspace's by then). Keying each mark on its own is what lets two tabs +// mark different sessions at the same time without one rewriting the other's mark away, as +// a single JSON blob would. + +interface PendingMarks { + dirty: { id: string; v: number }[] + removed: { id: string; ws?: string; key: string; storages?: string[] }[] +} + +function pendingBase(): string | undefined { + return scopedKey(PENDING_PREFIX) +} + +/** The marks of the user a write landed for: the current user unless the signal says + * otherwise (its store's user, when the user changed while the write was pending). */ +function pendingBaseFor(email: string | undefined): string | undefined { + return email ? scopedKeyFor(PENDING_PREFIX, email) : pendingBase() +} + +function dirtyKey(base: string, sessionId: string): string { + return `${base}::d::${sessionId}` +} + +function removedKey(base: string, sessionId: string, ws: string | undefined): string { + return `${base}::r::${sessionId}::${ws ?? ''}` +} + +function readPending(): PendingMarks { + const marks: PendingMarks = { dirty: [], removed: [] } + const base = pendingBase() + if (!base) return marks + try { + for (let i = 0; i < localStorage.length; i++) { + const key = localStorage.key(i) + if (!key || !key.startsWith(`${base}::`)) continue + const rest = key.slice(base.length + 2) + if (rest.startsWith('d::')) { + const v = Number(localStorage.getItem(key)) + marks.dirty.push({ id: rest.slice(3), v: Number.isFinite(v) ? v : 0 }) + } else if (rest.startsWith('r::')) { + const [id, ws] = rest.slice(3).split('::') + marks.removed.push({ id, ws: ws || undefined, key, storages: storagesOf(key) }) + } + } + } catch (e) { + console.error('Could not read session backup marks', e) + } + // Marks this page could not write: their rows carry the bumps, or this page does until + // a row exists (see `bumpViaSyncRow`); the counter counts with those. + for (const [id, bumps] of unwritableMarks) { + const mark = marks.dirty.find((d) => d.id === id) + if (mark) mark.v += bumps + else marks.dirty.push({ id, v: bumps }) + } + return marks +} + +/** False when the mark could not be written (storage full): the caller must carry the + * change some other way. */ +function bumpDirty(sessionId: string, email?: string): boolean { + const base = pendingBaseFor(email) + if (!base) return false + try { + const key = dirtyKey(base, sessionId) + const v = Number(localStorage.getItem(key) ?? '0') + localStorage.setItem(key, String((Number.isFinite(v) ? v : 0) + 1)) + return true + } catch (e) { + console.error('Could not persist session backup mark', e) + return false + } +} + +/** The durable fallback for a dirty mark that could not be written: the bump goes on the + * sync row, where a flush in flight cannot lose it (`writeSync` keeps it). A session with + * no row yet (its first push in flight, say) keeps it in this page until a row is written, + * which takes it over (see `writeSync`): the row the push writes would otherwise retire the + * mark with the bump unseen. Another user's session with no row is left to that user's next + * load, whose backfill marks every session without a row. */ +async function bumpViaSyncRow(id: string, email = getCurrentUserEmail()): Promise { + if (!email) return + let carried = false + await updateSyncRow(id, email, (row) => { + if (!row) return undefined + carried = true + return { ...row, extraV: (row.extraV ?? 0) + 1 } + }) + if (!carried && email === getCurrentUserEmail()) { + unwritableMarks.set(id, (unwritableMarks.get(id) ?? 0) + 1) + } +} + +/** Reads a row and writes what `update` makes of it, in the store of `email`: through the + * shared handle when that is the current user, and through a connection of its own + * otherwise, since the shared handle follows the current user and a write that landed + * after a switch must still reach the store it belongs to. */ +async function updateSyncRow( + id: string, + email: string, + update: (row: MirrorSyncState | undefined) => MirrorSyncState | undefined +): Promise { + if (email === getCurrentUserEmail()) { + const next = update(await readSync(id, email)) + if (next) await writeSync([next], email) + return + } + let db: IDBPDatabase | undefined + try { + db = await openDB(scopedKeyFor(SYNC_DB, email), 1, { upgrade: createSyncStore }) + const next = update(await db.get('sync', id)) + if (next) await db.put('sync', next) + } catch (e) { + console.error('Could not update the session backup state of another user', e) + } finally { + db?.close() + } +} + +/** Drop the dirty mark of a session gone from the store, the one case nothing can bump + * again. Every other mark stays: one whose push landed is retired through `flushedV` on + * the sync row (two localStorage calls cannot compare-and-delete, and a bump landing + * between them would be lost), and a draft's or an off workspace's waits its turn. */ +function dropDirty(sessionId: string): void { + const base = pendingBase() + if (base) removeKey(dirtyKey(base, sessionId)) +} + +/** The storages a removal mark names, when it does. */ +function storagesOf(key: string): string[] | undefined { + try { + const parsed: unknown = JSON.parse(localStorage.getItem(key) ?? '') + if (Array.isArray(parsed) && parsed.every((s) => typeof s === 'string')) return parsed + } catch {} + return undefined +} + +/** False when the mark could not be written (storage full): the caller must not act as if + * the removal were scheduled. */ +function addRemoved( + sessionId: string, + ws: string | undefined, + dropDirty: boolean, + email?: string, + storages?: string[] +): boolean { + const base = pendingBaseFor(email) + if (!base) return false + try { + if (dropDirty) localStorage.removeItem(dirtyKey(base, sessionId)) + localStorage.setItem( + removedKey(base, sessionId, ws), + storages && storages.length > 0 ? JSON.stringify(storages) : '1' + ) + return true + } catch (e) { + console.error('Could not persist session backup mark', e) + return false + } +} + +function removeKey(key: string): void { + try { + localStorage.removeItem(key) + } catch {} +} + +// --- Scheduling --- + +let quietTimer: ReturnType | undefined +let maxTimer: ReturnType | undefined +let retryTimer: ReturnType | undefined +let retryAt = 0 +let retryMs = RETRY_MIN_MS +/** + * Workspaces whose storage answered this page load. `off`: nowhere to keep backups. + * `refused`: the server rejected what this page sends; nothing more is sent for the page, + * but the marks and the sync state stay, so the next load tries again. + */ +const wsState = new Map() +/** Backups an admin turns on again elsewhere are noticed `OFF_RETRY_MS` after the workspace + * was found off: a timer per off workspace forgets the state then and asks again, so a + * pending mark left for the workspace does not wait for something else to flush. The + * admin's own page hears of the switch at once (see `backupSettingsChanged`). */ +const offTimers = new Map>() +const OFF_RETRY_MS = 10 * 60_000 +let offRetryMs = OFF_RETRY_MS + +function markOff(ws: string): void { + wsState.set(ws, 'off') + clearTimeout(offTimers.get(ws)) + offTimers.set( + ws, + setTimeout(() => { + offTimers.delete(ws) + if (wsState.get(ws) === 'off') wsState.delete(ws) + restoredWorkspaces.delete(ws) + void enqueue(flush) + restoreSessionBackups(ws) + }, offRetryMs) + ) +} + +function forgetOff(ws: string): void { + clearTimeout(offTimers.get(ws)) + offTimers.delete(ws) + if (wsState.get(ws) === 'off') wsState.delete(ws) +} + +function clearOffTimers(): void { + for (const timer of offTimers.values()) clearTimeout(timer) + offTimers.clear() +} + +function isOff(ws: string): boolean { + return wsState.get(ws) === 'off' +} +let backfilled = false +/** Sessions of the current user whose dirty mark could not be written this page. */ +/** Sessions whose dirty mark localStorage refused this page, with the bumps of theirs no + * sync row could take yet. */ +let unwritableMarks = new Map() +const restoredWorkspaces = new Set() + +// One flush or restore at a time in this tab; each reads the marks fresh. +let chain: Promise = Promise.resolve() +function enqueue(fn: () => Promise): Promise { + const run = chain.then(fn, fn) + chain = run.catch((e) => console.error('Session backup failed', e)) + return run +} + +function clearTimers(): void { + clearTimeout(quietTimer) + clearTimeout(maxTimer) + quietTimer = undefined + maxTimer = undefined +} + +function scheduleFlush(): void { + clearTimeout(quietTimer) + quietTimer = setTimeout(runFlush, QUIET_MS) + maxTimer ??= setTimeout(runFlush, MAX_DELAY_MS) +} + +function runFlush(): void { + clearTimers() + void enqueue(flush) +} + +function backOff(): void { + retryAt = Date.now() + retryMs + retryMs = Math.min(retryMs * 2, RETRY_MAX_MS) + clearTimeout(retryTimer) + retryTimer = setTimeout(runFlush, retryAt - Date.now()) +} + +/** Serialize with the other tabs of the same user where the browser lets us; on plain + * http there is no lock, and two tabs at worst upload the same bytes twice. */ +/** One tab of the user at a time in the flush and the restore. A flush finding the lock + * taken reschedules itself; a restore waits its turn, since two tabs restoring the same + * absent session would each write its pieces over the other's. */ +function webLocks(): LockManager | undefined { + return typeof navigator === 'undefined' ? undefined : (navigator as { locks?: LockManager }).locks +} + +function hasWebLocks(): boolean { + return webLocks() !== undefined +} + +async function withUserLock(email: string, fn: () => Promise, wait = false): Promise { + const locks = webLocks() + if (!locks) return fn() + await locks.request(`wm-ai-sessions-mirror::${email}`, { ifAvailable: !wait }, async (lock) => { + if (lock) await fn() + // The other tab's flush read the marks before this one's were written: try again + // once it is done, rather than wait for the next write or load. + else scheduleFlush() + }) +} + +// --- Flush --- + +function statusOf(e: unknown): number | undefined { + return e instanceof ApiError ? e.status : undefined +} + +async function readSync(id: string, email: string): Promise { + return (await syncDb(email))?.get('sync', id) +} + +async function allSyncRows(email: string): Promise { + return (await (await syncDb(email))?.getAll('sync')) ?? [] +} + +/** The durable fallback for a user delete whose localStorage mark could not be written. + * A session with a row was backed up; one without may have its first push in flight, so + * it gets a row saying only that, which the push's own row write keeps (`writeSync`). An + * unsent draft gets nothing: it was never pushed. */ +async function removeViaSyncRow( + id: string, + ws: string | undefined, + email = getCurrentUserEmail() +): Promise { + if (!email) return + await updateSyncRow(id, email, (row) => + row + ? { ...row, removed: true } + : ws + ? { id, ws, head: '', chats: {}, images: {}, removed: true } + : undefined + ) +} + +/** Writes rows whole, except that a removal filed on a row meanwhile survives: the flush + * writes a session's row from state it read before the push, and the user may have deleted + * the session in between. */ +/** A row that says where else the copy is (`alsoIn`) is believed; one that does not + * inherits it, plus the storage the row it replaces was on when that is another one of this + * workspace's: the copy there stays where it was. */ +async function writeSync(states: MirrorSyncState[], email: string): Promise { + const db = await syncDb(email) + if (!db || states.length === 0) return + // Bumps this page kept for want of a row (see `bumpViaSyncRow`) move onto the row now; + // what arrives while the write is in flight stays counted here. + const taken = new Map() + const tx = db.transaction('sync', 'readwrite') + for (const s of states) { + const cur = await tx.store.get(s.id) + const removed = s.removed || cur?.removed + const kept = email === getCurrentUserEmail() ? (unwritableMarks.get(s.id) ?? 0) : 0 + if (kept > 0) taken.set(s.id, kept) + const extraV = Math.max(s.extraV ?? 0, cur?.extraV ?? 0) + kept + const alsoIn = new Set(s.alsoIn ?? []) + if (s.alsoIn === undefined && cur?.ws === s.ws) { + for (const id of cur.alsoIn ?? []) alsoIn.add(id) + if (cur.storageId !== undefined) alsoIn.add(cur.storageId) + } + if (s.storageId !== undefined) alsoIn.delete(s.storageId) + const { alsoIn: _, ...rest } = s + await tx.store.put({ + ...rest, + ...(removed ? { removed: true } : {}), + ...(extraV > 0 ? { extraV } : {}), + ...(alsoIn.size > 0 ? { alsoIn: [...alsoIn] } : {}) + }) + } + await tx.done + for (const [id, n] of taken) unwritableMarks.set(id, (unwritableMarks.get(id) ?? 0) - n) +} + +async function deleteSync(ids: string[], email: string): Promise { + const db = await syncDb(email) + if (!db || ids.length === 0) return + const tx = db.transaction('sync', 'readwrite') + for (const id of ids) await tx.store.delete(id) + await tx.done +} + +/** A workspace's storage went away: what was pushed there can no longer be trusted to be + * where a later storage looks, so every session goes whole on the next push. The rows stay + * (stale) so a removal still knows a backup existed. */ +async function staleWorkspaceSync(ws: string, email: string): Promise { + const db = await syncDb(email) + if (!db) return + const rows = (await db.getAll('sync')).filter((s) => s.ws === ws && !s.stale) + await writeSync( + rows.map((s) => ({ ...s, stale: true })), + email + ) +} + +/** The workspace's live rows recorded against another storage than the one the server + * answers from: they describe objects it no longer looks at (a new bucket starts empty). */ +function foreignRows( + ws: string, + storageId: string, + generation: number, + rows: Iterable +): MirrorSyncState[] { + return [...rows].filter( + (row) => + row.ws === ws && + !row.stale && + !row.removed && + !row.staging && + (row.storageId !== storageId || (row.generation ?? 0) !== generation) + ) +} + +/** Stale rows plan like no row at all, and the mark makes the next flush pick them up. */ +async function markStale(rows: MirrorSyncState[], email: string): Promise { + await writeSync( + rows.map((row) => ({ ...row, stale: true })), + email + ) + for (const row of rows) bumpDirty(row.id) +} + +/** Sessions this browser has backed up nothing of yet, whose backup went stale, or whose + * row carries bumps (localStorage refused their marks) no push has covered: everything + * committed to a workspace gets a mark, once per page load. */ +async function backfillMarks(marks: PendingMarks, email: string): Promise { + if (backfilled) return true + const sessions = await readStoredSessions(email) + const db = await syncDb(email) + if (!sessions || !db) return false + backfilled = true + const rows = new Map((await db.getAll('sync')).map((s) => [s.id, s])) + const marked = new Set(marks.dirty.map((d) => d.id)) + for (const s of sessions) { + const row = rows.get(s.id) + // A restored row has neither counter: only bumps it carries make it owed. + const owed = + !row || row.stale || ((row.extraV ?? 0) > 0 && (row.flushedV ?? -1) < (row.extraV ?? 0)) + if (s.workspace_id && owed && !marked.has(s.id)) { + // The counter as the next `readPending` will see it: the mark's, or this page's + // when none could be written, so what this flush retires is what a later bump + // counts from. + let v = 1 + if (!bumpDirty(s.id)) { + if (!unwritableMarks.has(s.id)) unwritableMarks.set(s.id, 0) + v = unwritableMarks.get(s.id) ?? 0 + } + marks.dirty.push({ id: s.id, v }) + } + } + return true +} + +/** Read what the stores hold for a dirty session and plan its push. `undefined` when the + * session has nowhere to go (unsent), `unavailable` when a store could not be read. */ +async function planFor( + session: Session, + sync: MirrorSyncState | undefined, + email: string +): Promise { + const chatIds = await listSessionChatIds(session.id, email) + if (!chatIds) return 'unavailable' + const chats: ChatSnapshot[] = [] + for (const id of chatIds) { + const record = await readStoredChat(id, email) + if (!record) continue + const imageIds = (await listChatImageIds(id, email)) ?? [] + if (jsonBytes(record) > MAX_CHAT_BYTES) { + console.warn(`AI session chat ${id} is too large to back up; leaving it out`) + chats.push({ id, lastModified: record.lastModified, imageIds, omitted: true }) + continue + } + chats.push({ id, lastModified: record.lastModified, record, imageIds }) + } + let artifacts = await readSessionArtifacts(session.id, email) + if (!artifacts) return 'unavailable' + if (jsonBytes(artifacts) > MAX_ARTIFACTS_BYTES) { + console.warn(`AI session ${session.id} artifacts are too large to back up; leaving them out`) + artifacts = { items: [], versions: [] } + } + return planSessionPush({ session, chats, artifacts, sync }) +} + +interface WorkspaceWork { + items: { session: Session; v: number; sync?: MirrorSyncState }[] + /** `key` is the localStorage mark; absent when the removal rides on the sync row. + * `storageId` and `alsoIn` are the storages holding a copy, when a sync row says: a + * removal is done only once each of them answered it. */ + removed: { id: string; key?: string; storageId?: string; alsoIn?: string[] }[] +} + +type SendStatus = 'ok' | 'off' | 'refused' | 'abort' | 'transient' + +interface WorkspaceOutcome { + status: SendStatus + /** The storage and backup generation the server answered from, once it answered. */ + storageId?: string + generation?: number + /** Sessions every part of which the server stored. */ + settled: { + id: string + v: number + next: MirrorSyncState + removeFrom?: string + carried: boolean + storageId?: string + generation?: number + }[] + /** Marks with nothing behind them (an unsent draft, a session gone from the store). */ + dropped: { id: string; v: number }[] + /** Sessions the server holds no head of any more (another device removed the backup): + * their rows go stale, so the next flush sends them whole. */ + needsWhole: string[] + /** Removal marks the server carried out. */ + removedDone: { id: string; key?: string }[] + /** Removals one storage carried out while others still hold a copy: the row narrows to + * those, and the mark waits for them to answer. */ + removedFrom: { id: string; key?: string; remaining: string[] }[] + /** Some session the server could not store. */ + anyFailed: boolean + /** Some store could not be read; its marks stay for a later flush. */ + unavailable: boolean +} + +/** + * Plan and send a workspace's sessions one at a time, filling requests of about + * REQUEST_TARGET_BYTES as it goes, so a first backfill never holds more than one + * request's worth of records and images at once. Removals go first, in their own + * request(s). + */ +async function pushWorkspace( + ws: string, + work: WorkspaceWork, + email: string +): Promise { + const out: WorkspaceOutcome = { + status: 'ok', + settled: [], + dropped: [], + needsWhole: [], + removedDone: [], + removedFrom: [], + anyFailed: false, + unavailable: false + } + // Parts of a session still to be acknowledged, and the sessions the server refused. + // `complete` once every part of the session has been appended: a session whose first + // part is still to come when a request fails has nothing behind it yet. + const attempted = new Map< + string, + { + v: number + next: MirrorSyncState + parts: number + complete: boolean + removeFrom?: string + carried: boolean + storageId?: string + generation?: number + /** The storage and generation the last answer for this session came from. */ + answered?: string + } + >() + const failed = new Set() + const headless = new Set() + let current: PushBody | undefined + let size = 0 + let ops = 0 + + const send = async (body: PushBody): Promise => { + if (getCurrentUserEmail() !== email) return 'abort' + let res + try { + res = await AiService.pushAiSessionBackups({ workspace: ws, requestBody: body }) + } catch (e) { + const status = statusOf(e) + // 404: a build without object storage. 403: nothing this token may back up. + if (status === 404 || status === 403) return 'off' + if (status === 409) return 'abort' + // Too large is a fact about the sessions in this body, not the workspace: they + // stay marked (and retried with backoff), the others go on. + if (status === 413) { + console.error('Session backup push too large', e) + for (const entry of body.sessions) failed.add(entry.id) + for (const id of body.removed ?? []) failed.add(id) + return 'ok' + } + if (status !== undefined && status < 500 && status !== 429) { + console.error('Session backup push refused', e) + return 'refused' + } + console.warn('Session backup push failed, retrying later', e) + return 'transient' + } + if (!res.enabled) return 'off' + out.storageId = res.storage_id + out.generation = res.backup_generation + const answered = `${res.storage_id}:${res.backup_generation}` + const errors = new Set() + for (const r of res.results) { + if (r.error) { + console.warn(`Session backup of ${r.id} failed: ${r.error}`) + errors.add(r.id) + } else if (r.needs_whole) { + // The part rode on a head the storage no longer has: not a failure to back + // off from, but nothing to settle either. + headless.add(r.id) + out.needsWhole.push(r.id) + } + } + for (const entry of body.sessions) { + const a = attempted.get(entry.id) + if (!a) continue + a.parts -= 1 + // Parts answered from different storages sit in different buckets: nothing to + // settle, the session goes again whole. + if (a.answered !== undefined && a.answered !== answered) failed.add(entry.id) + a.answered = answered + a.storageId = res.storage_id + a.generation = res.backup_generation + if (errors.has(entry.id)) failed.add(entry.id) + } + for (const id of body.removed ?? []) { + if (errors.has(id)) failed.add(id) + else { + const mark = work.removed.find((r) => r.id === id) + if (!mark) continue + const holding = new Set( + [mark.storageId, ...(mark.alsoIn ?? [])].filter((s): s is string => s !== undefined) + ) + // Answered from a storage holding no copy: the copies are still where they + // were, and the mark waits for those storages to answer. + if (holding.size === 0 || res.storage_id === undefined) out.removedDone.push(mark) + else if (holding.has(res.storage_id)) { + holding.delete(res.storage_id) + if (holding.size === 0) out.removedDone.push(mark) + else out.removedFrom.push({ id, key: mark.key, remaining: [...holding] }) + } + } + } + return 'ok' + } + const flushCurrent = async (): Promise => { + if (!current) return 'ok' + const body = current + current = undefined + size = 0 + ops = 0 + return send(body) + } + const append = async (entry: AISessionBackupPush): Promise => { + const bytes = jsonBytes(entry) + const entryOps = operationsOf(entry) + if ( + current && + (current.sessions.length >= MAX_ENTRIES_PER_REQUEST || + ops + entryOps > MAX_OPERATIONS_PER_REQUEST || + size + bytes > REQUEST_TARGET_BYTES) + ) { + const status = await flushCurrent() + if (status !== 'ok') return status + } + // A part the server refused holds the rest of its session back: the head rides on + // the last part, and would list a session missing a piece. + if (failed.has(entry.id)) return 'ok' + current ??= { owner: email, sessions: [] } + current.sessions.push(entry) + size += bytes + ops += entryOps + const a = attempted.get(entry.id) + if (a) a.parts += 1 + return 'ok' + } + const finish = (status: SendStatus): WorkspaceOutcome => { + out.status = status + for (const [id, a] of attempted) { + if (a.complete && a.parts === 0 && !failed.has(id) && !headless.has(id)) { + out.settled.push({ + id, + v: a.v, + next: a.next, + removeFrom: a.removeFrom, + carried: a.carried, + storageId: a.storageId, + generation: a.generation + }) + } + } + out.anyFailed = failed.size > 0 + return out + } + + for (let i = 0; i < work.removed.length; i += MAX_REMOVED_PER_REQUEST) { + const chunk = work.removed.slice(i, i + MAX_REMOVED_PER_REQUEST) + const status = await send({ owner: email, sessions: [], removed: chunk.map((r) => r.id) }) + if (status !== 'ok') return finish(status) + } + + for (const item of work.items) { + if (getCurrentUserEmail() !== email) return finish('abort') + const plan = await planFor(item.session, item.sync, email) + if (plan === 'unavailable') { + out.unavailable = true + continue + } + if (!plan) { + out.dropped.push({ id: item.session.id, v: item.v }) + continue + } + const nothingToSend = !plan.entry && plan.images.length === 0 + // A move: the copy in the old workspace is filed for removal once this push has + // landed (see `settled`), never before, so the session is backed up somewhere at + // every point. + // A session with nothing to send gets no answer to name its storage: it keeps the + // one its row has, which the storage check below then judges like any other. + attempted.set(item.session.id, { + v: item.v, + next: plan.next, + parts: 0, + complete: nothingToSend, + removeFrom: plan.removeFrom, + carried: plan.carried, + storageId: item.sync?.storageId, + generation: item.sync?.generation + }) + if (nothingToSend) continue + // A push of the session whole names itself on every part and opens with its head on + // the first, whichever that is: the server replaces the backup on that part, lists + // the session by the last, and meanwhile refuses any other push of it. + const epoch = item.session.moves ?? 0 + let opened = false + let token: string | undefined + const open = (part: AISessionBackupPush): AISessionBackupPush => { + const first = !opened + opened = true + // A push split over parts (the first says more follow) names itself on each, so + // the server keeps the session unlisted between them; a whole one opens with the + // head on whichever part goes first. + if (first && part.partial) token = randomUUID() + return { + ...part, + epoch, + ...(plan.whole ? { whole: true } : {}), + ...(plan.whole && first ? { head: plan.entry?.head } : {}), + ...(token ? { push: token } : {}), + ...(token && first ? { opens: true } : {}) + } + } + let images: AISessionBackupImage[] = [] + let imagesBytes = 0 + for (const { chat_id, id } of plan.images) { + const data_url = await readImageDataUrl(id, email) + // Evicted since the plan was made; the next save of that chat drops the id. + if (!data_url) continue + if ( + images.length > 0 && + (images.length >= MAX_IMAGES_PER_ENTRY || + imagesBytes + data_url.length > REQUEST_TARGET_BYTES) + ) { + const status = await append(open({ id: item.session.id, images, partial: true })) + if (status !== 'ok') return finish(status) + images = [] + imagesBytes = 0 + } + images.push({ chat_id, id, data_url }) + imagesBytes += data_url.length + } + // Every part but the last says so: the server lists a session on the part that + // completes its entry, never on one an unsent part still follows. + const parts = plan.entry + ? splitEntry( + plan.whole ? { ...plan.entry, head: undefined } : plan.entry, + REQUEST_TARGET_BYTES + ) + : [] + if (images.length > 0) { + const status = await append( + open({ + id: item.session.id, + images, + partial: parts.length > 0 || undefined + }) + ) + if (status !== 'ok') return finish(status) + } + for (const [i, part] of parts.entries()) { + if (failed.has(item.session.id)) break + const status = await append(open(i < parts.length - 1 ? { ...part, partial: true } : part)) + if (status !== 'ok') return finish(status) + } + attempted.get(item.session.id)!.complete = true + } + return finish(await flushCurrent()) +} + +async function flush(): Promise { + const email = getCurrentUserEmail() + if (!email) return + if (Date.now() < retryAt) { + clearTimeout(retryTimer) + retryTimer = setTimeout(runFlush, retryAt - Date.now()) + return + } + await withUserLock(email, async () => { + const marks = readPending() + // A store that cannot be opened right now (another tab's upgrade in progress) is + // retried with backoff, as any other unavailable store below. + if (!(await backfillMarks(marks, email))) { + backOff() + return + } + // Read once: retired marks are not reclaimed (a mark cannot be deleted without a + // window in which a bump is lost), so there is one per session ever backed up, and + // telling them apart from live ones is what lets a flush with nothing to do stop + // here, before the sessions store. + const syncRows = new Map((await allSyncRows(email)).map((row) => [row.id, row])) + // A mark's counter counts with the bumps its row carries (the ones localStorage refused). + const live = marks.dirty + .map((d) => ({ id: d.id, v: d.v + (syncRows.get(d.id)?.extraV ?? 0) })) + .filter((d) => { + const sync = syncRows.get(d.id) + return !(sync && !sync.stale && (sync.flushedV ?? -1) >= d.v) + }) + // A removal whose mark could not be written rides on the sync row instead. + const removals: { id: string; ws?: string; key?: string; storages?: string[] }[] = [ + ...marks.removed + ] + for (const row of syncRows.values()) { + if (row.removed && !removals.some((r) => r.id === row.id)) { + removals.push({ id: row.id, ws: row.ws }) + } + } + if (live.length === 0 && removals.length === 0) return + const stored = await readStoredSessions(email) + if (!stored) { + backOff() + return + } + const byId = new Map(stored.map((s) => [s.id, s])) + const work = new Map() + const workFor = (ws: string) => { + let w = work.get(ws) + if (!w) work.set(ws, (w = { items: [], removed: [] })) + return w + } + const droppedDirty: string[] = [] + const consumedRemoved: string[] = [] + + for (const r of removals) { + const sync = syncRows.get(r.id) + const ws = r.ws ?? sync?.ws + // Nowhere to remove it from (an unsent draft): done. A workspace whose backups are + // off keeps the removal of a session that was backed up, for when they are on + // again, or the session would come back; one never backed up from here has + // nothing there, so its mark goes, or a storage-less instance would collect one + // per deleted session forever. + if (!ws) { + if (r.key) consumedRemoved.push(r.key) + } else if (!isOff(ws) && wsState.get(ws) !== 'refused') { + // The row says where the copy is only for its own workspace: a session that + // moved on has the row its new workspace wrote, and its mark says instead. + const own = sync?.ws === ws ? sync : undefined + const holding = (own ? [own.storageId, ...(own.alsoIn ?? [])] : (r.storages ?? [])).filter( + (x): x is string => x !== undefined + ) + workFor(ws).removed.push({ + id: r.id, + key: r.key, + storageId: holding[0], + alsoIn: holding.slice(1) + }) + } else if (isOff(ws) && !sync && r.key) consumedRemoved.push(r.key) + } + for (const d of live) { + const session = byId.get(d.id) + // Gone from the store, so nothing can bump it again; an unsent draft keeps its + // mark, since it may commit to a workspace while this flush runs. + if (!session) { + droppedDirty.push(d.id) + continue + } + if (session.transient || !session.workspace_id) continue + // Left in place for a workspace that is off or refused: a move into it must still + // remember the old copy, and a mark costs one lookup per flush. + if (isOff(session.workspace_id) || wsState.get(session.workspace_id) === 'refused') { + continue + } + const sync = syncRows.get(d.id) + // A stale row plans like no row at all: the whole session goes again. One naming + // another workspace still says where the old copy is, stale or not. + workFor(session.workspace_id).items.push({ + session, + v: d.v, + sync: sync?.stale && sync.ws === session.workspace_id ? undefined : sync + }) + } + + let settledAny = false + let leftForNext = false + for (const [ws, w] of work) { + const out = await pushWorkspace(ws, w, email) + if (out.status === 'abort') return + if (out.status === 'off') { + markOff(ws) + await staleWorkspaceSync(ws, email) + // Same rule as the loop above: a removal is worth keeping only for a session + // that was backed up from here. + for (const r of w.removed) { + if (!syncRows.has(r.id) && r.key) consumedRemoved.push(r.key) + } + continue + } + // Refused stops the workspace for the page, but what the earlier requests of this + // flush stored is recorded like any other. + if (out.status === 'refused') wsState.set(ws, 'refused') + if (out.status === 'transient' || out.anyFailed || out.unavailable) backOff() + else settledAny = true + // The new workspace holds a moved session now, so the old copy can go: its removal + // mark is written before the row that forgets where the old copy was, and a + // session whose mark could not be written keeps its old row, so the next flush + // plans the move again rather than orphan the copy. + const recorded = out.settled.filter((s) => { + if (s.removeFrom) { + const old = syncRows.get(s.id) + const storages = + old?.ws === s.removeFrom + ? [old.storageId, ...(old.alsoIn ?? [])].filter((x): x is string => x !== undefined) + : undefined + if (!addRemoved(s.id, s.removeFrom, false, undefined, storages)) return false + } + if (s.removeFrom || s.carried) leftForNext = true + return true + }) + // A session with deletes carried over stays one bump short of retired, so the + // next flush sends the rest. + const written = recorded.map((s) => ({ + ...s.next, + flushedV: s.carried ? s.v - 1 : s.v, + storageId: s.storageId, + generation: s.generation + })) + await writeSync(written, email) + // The server names the storage and generation every answer comes from. Rows + // naming another go stale, the ones written just now included: a session answered + // from a storage the later answers left behind, or pushed in part on top of a row + // from another one, has its backup split across buckets the server no longer + // looks at as a whole. + if (out.storageId !== undefined) { + const storageId = out.storageId + const generation = out.generation ?? 0 + const elsewhere = (row: MirrorSyncState) => + row.storageId !== storageId || (row.generation ?? 0) !== generation + const removed = new Set(out.removedDone.map((r) => r.id)) + const writtenIds = new Set(written.map((row) => row.id)) + const untouched = [...syncRows.values()].filter( + (row) => !removed.has(row.id) && !writtenIds.has(row.id) + ) + const foreign = [ + ...foreignRows(ws, storageId, generation, untouched), + ...written.filter((row) => { + const prior = syncRows.get(row.id) + const partial = prior && !prior.stale && prior.ws === ws && elsewhere(prior) + return elsewhere(row) || partial + }) + ] + if (foreign.length > 0) { + await markStale(foreign, email) + leftForNext = true + } + } + droppedDirty.push(...out.dropped.map((d) => d.id)) + // The server holds no head of these any more (another device removed the backup): + // stale rows send them whole next. + const headless = out.needsWhole + .map((id) => syncRows.get(id)) + .filter((row): row is MirrorSyncState => row !== undefined && !row.stale) + if (headless.length > 0) { + await markStale(headless, email) + leftForNext = true + } + for (const r of out.removedDone) { + // The row describes this workspace's copy only; a session that moved on keeps + // the row its new workspace wrote. + if ((await readSync(r.id, email))?.ws === ws) await deleteSync([r.id], email) + if (r.key) consumedRemoved.push(r.key) + } + for (const r of out.removedFrom) { + const row = await readSync(r.id, email) + const [storageId, ...alsoIn] = r.remaining + if (row?.ws === ws) await writeSync([{ ...row, storageId, alsoIn }], email) + else if (r.key) { + try { + localStorage.setItem(r.key, JSON.stringify(r.remaining)) + } catch {} + } + } + } + if (settledAny && Date.now() >= retryAt) retryMs = RETRY_MIN_MS + if (getCurrentUserEmail() !== email) return + for (const id of droppedDirty) dropDirty(id) + for (const key of consumedRemoved) removeKey(key) + // Whatever is still marked is either waiting on the backoff timer, on a write that + // scheduled its own flush, or on a workspace that is off for a while or refused for the page; + // none of it wants another flush in 15 s. What this flush left for the next one + // does: a moved session's removal from its old workspace, carried-over deletes, or + // the sessions of a storage the server no longer answers from. + if (leftForNext) scheduleFlush() + }) +} + +// --- Restore --- + +function asRecord(value: unknown): Record | undefined { + return value && typeof value === 'object' && !Array.isArray(value) + ? (value as Record) + : undefined +} + +/** Turn one pulled backup into store rows, dropping anything that does not name this + * session: the bucket is written by the server from validated ids, but a record is + * still data from outside this browser. */ +function unpackBackup( + ws: string, + backup: AISessionBackup, + updatedAt: number, + storageId: string | undefined, + generation: number | undefined, + earlierChats: Iterable = [] +): + | { + session: Session + chats: StoredChat[] + images: RestoredImage[] + artifacts: { items: PersistedArtifact[]; versions: ArtifactVersion[] } + sync: MirrorSyncState + } + | undefined { + const head = asRecord(backup.head) + if (!head || head.id !== backup.id || head.workspace_id !== ws) return undefined + if (typeof head.createdAt !== 'number') return undefined + const chats: StoredChat[] = [] + for (const c of backup.chats) { + const record = asRecord(c.record) + if ( + !record || + record.id !== c.id || + record.sessionId !== backup.id || + !Array.isArray(record.actualMessages) || + !Array.isArray(record.displayMessages) || + typeof record.lastModified !== 'number' + ) { + continue + } + chats.push(record as unknown as StoredChat) + } + // An image's chat may have come on an earlier page of the session. + const chatIds = new Set([...chats.map((c) => c.id), ...earlierChats]) + const images: RestoredImage[] = backup.images + .filter((i) => chatIds.has(i.chat_id) && typeof i.data_url === 'string') + .map((i) => ({ id: i.id, chatId: i.chat_id, dataUrl: i.data_url })) + const artifactsRecord = asRecord(backup.artifacts) + const items = ( + Array.isArray(artifactsRecord?.items) ? (artifactsRecord.items as PersistedArtifact[]) : [] + ).filter((i) => asRecord(i)?.sessionId === backup.id && typeof i.id === 'string') + const itemIds = new Set(items.map((i) => i.id)) + const versions = ( + Array.isArray(artifactsRecord?.versions) ? (artifactsRecord.versions as ArtifactVersion[]) : [] + ).filter((v) => asRecord(v) && itemIds.has(v.artifactId) && typeof v.key === 'string') + const active = chats.find((c) => c.id === head.chatId) + const session: Session = { + ...(head as unknown as Session), + name: '', + // Everything in the backup has been read here: no unread badge, and the last + // activity is the backup's own time. + lastSeenCount: active?.displayMessages.length ?? 0, + lastActivityAt: updatedAt + } + const sync: MirrorSyncState = { + id: backup.id, + ws, + head: headSig(session), + chats: Object.fromEntries(chats.map((c) => [c.id, c.lastModified])), + images: Object.fromEntries(images.map((i) => [i.id, i.chatId])), + artifacts: artifactsFingerprint({ items, versions }), + storageId, + generation + } + return { session, chats, images, artifacts: { items, versions }, sync } +} + +type BackupListing = Awaited> + +/** The workspace's listing; `off` when it keeps no backups, `failed` when it could not be + * listed this time. */ +async function listWorkspace(ws: string, email: string): Promise { + let listing + try { + listing = await AiService.listAiSessionBackups({ workspace: ws }) + } catch (e) { + const status = statusOf(e) + if (status === 404 || status === 403) { + markOff(ws) + return 'off' + } + console.warn('Could not list session backups', e) + return 'failed' + } + if (!listing.enabled) { + markOff(ws) + return 'off' + } + if (wsState.get(ws) !== 'refused') wsState.set(ws, 'on') + const foreign = + listing.storage_id === undefined + ? [] + : foreignRows( + ws, + listing.storage_id, + listing.backup_generation ?? 0, + await allSyncRows(email) + ) + if (foreign.length > 0) { + await markStale(foreign, email) + scheduleFlush() + } + return listing +} + +/** The workspaces of one family are restored together: a session moved between two of them + * is listed by both until the old copy's removal lands (that mark is the moving browser's, + * which may never come back), and whichever were restored first would take the id and keep + * the other out. The copy that moved last (`epoch`, the record's move count, kept with the + * marker) is the one brought back, the storage's own modification time deciding between two + * of the same count; the other is left where it is. A family one of whose workspaces could + * not be listed is not restored at all this time, or the copy that lists could be the stale + * one; the next page load or workspace switch tries again. */ +async function restoreFamily(family: string[], todo: string[], email: string): Promise { + const listings = new Map() + for (const ws of todo) { + if (getCurrentUserEmail() !== email) return + const listing = await listWorkspace(ws, email) + if (listing === 'failed') { + for (const w of todo) restoredWorkspaces.delete(w) + return + } + if (listing !== 'off') listings.set(ws, listing) + // Probed again once its backups may be on again (see `isOff`). + else restoredWorkspaces.delete(ws) + } + const newest = new Map() + for (const [ws, listing] of listings) { + for (const s of listing.sessions) { + const at = Date.parse(s.updated_at) + const cur = newest.get(s.id) + if (!cur || s.epoch > cur.epoch || (s.epoch === cur.epoch && at > cur.at)) { + newest.set(s.id, { ws, epoch: s.epoch, at }) + } + } + } + // A move landing between the listings and the pulls (another device pushing the session + // into a workspace listed before it held it, or one whose backups were off then) would + // make a copy about to be imported the stale one: the whole family is listed again once + // a workspace's pulls are done and its records are about to land, and a session a later + // copy of which showed up elsewhere is left, with the family, for next time. A family of + // one has nowhere else for a copy to show up. + const verify = async (ids: string[]): Promise> => { + const superseded = new Set() + if (family.length < 2) return superseded + for (const w of family) { + const listing = await listWorkspace(w, email) + if (listing === 'failed') { + for (const id of ids) superseded.add(id) + break + } + if (listing === 'off') continue + for (const s of listing.sessions) { + const cur = newest.get(s.id) + if (!cur || !ids.includes(s.id) || w === cur.ws) continue + const at = Date.parse(s.updated_at) + if (s.epoch > cur.epoch || (s.epoch === cur.epoch && at > cur.at)) superseded.add(s.id) + } + } + if (superseded.size > 0) for (const w of todo) restoredWorkspaces.delete(w) + return superseded + } + for (const [ws, listing] of listings) { + if (getCurrentUserEmail() !== email) return + const elsewhere = listing.sessions.filter((s) => newest.get(s.id)?.ws !== ws).map((s) => s.id) + await restoreWorkspace(ws, email, listing, new Set(elsewhere), verify) + } +} + +async function restoreWorkspace( + ws: string, + email: string, + listing: BackupListing, + elsewhere: Set, + verify: (ids: string[]) => Promise> +): Promise { + const rows = await allSyncRows(email) + const local = new Set() + for (const s of (await readStoredSessions(email)) ?? []) local.add(s.id) + for (const s of sessionState.sessions) local.add(s.id) + // A removal names the workspace it is for: a session that moved here from another one + // still has that one's removal pending, and is ours to restore. + for (const r of readPending().removed) if (r.ws === ws) local.add(r.id) + for (const row of rows) if (row.removed && row.ws === ws) local.add(row.id) + const candidates = listing.sessions + .filter((s) => !local.has(s.id) && !elsewhere.has(s.id) && !isSessionTombstoned(s.id)) + .slice(0, RESTORE_MAX) + const updatedAt = new Map(candidates.map((s) => [s.id, Date.parse(s.updated_at)])) + let ids = candidates.map((s) => s.id) + const toImport: Staged[] = [] + // A session that did not fit one answer whole comes in pages, kept here until the last + // one: importing a page alone would leave a session the next restore takes for whole. + type Pieces = { + chats: Set + images: Set + items: Set + versions: Set + } + type Staged = { + session: Session + sync: MirrorSyncState + /** Everything the pages so far wrote for the session. */ + pieces: Pieces + /** The listing fingerprint the pages so far were answered with. */ + listing?: string + } + const staged = new Map() + const noPieces = (): Pieces => ({ + chats: new Set(), + images: new Set(), + items: new Set(), + versions: new Set() + }) + const union = (a: Pieces, b: Pieces): Pieces => ({ + chats: new Set([...a.chats, ...b.chats]), + images: new Set([...a.images, ...b.images]), + items: new Set([...a.items, ...b.items]), + versions: new Set([...a.versions, ...b.versions]) + }) + // What earlier attempts (a restore cut short, a start over) wrote for a session that has + // no record yet, from their staging rows: the pieces of it the backup no longer has go + // before the record lands, or a later flush would push them back. Ids, never clocks. + const earlierStaging = new Map() + for (const row of rows) { + if (row.staging) { + earlierStaging.set(row.id, { + chats: new Set(row.staging.chats), + images: new Set(row.staging.images), + items: new Set(row.staging.items), + versions: new Set(row.staging.versions) + }) + } + } + // A session whose backup moved between two of its pages starts over, a few times. + const restarts = new Map() + const MAX_RESTARTS = 3 + const resumes: AISessionBackupCursor[] = [] + while (ids.length > 0 || resumes.length > 0) { + if (getCurrentUserEmail() !== email) return + const resume = resumes.shift() + const batch = resume ? [resume.id] : ids.slice(0, PULL_BATCH) + if (!resume) ids = ids.slice(PULL_BATCH) + let pulled + try { + pulled = await AiService.pullAiSessionBackups({ + workspace: ws, + requestBody: { ids: batch, resume } + }) + } catch (e) { + console.warn('Could not pull session backups', e) + return + } + if (!pulled.enabled) { + markOff(ws) + return + } + // Ask again for what did not fit, one at a time so each answer is as small as can be. + for (const id of pulled.deferred) if (!ids.includes(id)) ids.unshift(id) + // A session's pieces land before its record, page by page (the writes are absent-only, + // so a restore cut short leaves nothing a later one cannot finish), and the record, + // which is what makes the session visible, only with the last page; a session whose + // pieces could not be written is left for the next restore, since recording it now + // would let the next flush push its half-empty local state over the backup. What a + // page leaves for the next is the sync row being assembled, never its pieces. + const ready: Staged[] = [] + for (const b of pulled.sessions) { + const earlier = staged.get(b.id) + staged.delete(b.id) + // Brought back meanwhile by this tab itself (a session moved here from another + // workspace, say; the lock keeps other tabs out): its pieces are not ours to write + // over any more. + if ((await readStoredSessions(email))?.some((s) => s.id === b.id)) continue + // The backup moved between two pages (a chat sorting before the cursor would be + // missed) or under this one (the page may mix two versions): the pages so far do + // not belong together, the session starts over. + if (b.moved || (earlier && b.listing !== earlier.listing)) { + const n = (restarts.get(b.id) ?? 0) + 1 + restarts.set(b.id, n) + if (earlier) { + earlierStaging.set(b.id, union(earlierStaging.get(b.id) ?? noPieces(), earlier.pieces)) + } + if (n < MAX_RESTARTS) ids.unshift(b.id) + else console.warn(`Session backup ${b.id} kept changing while restoring; left for later`) + continue + } + const u = unpackBackup( + ws, + b, + updatedAt.get(b.id) ?? Date.now(), + pulled.storage_id, + pulled.backup_generation, + Object.keys(earlier?.sync.chats ?? {}) + ) + if (!u) continue + const written: Pieces = { + chats: new Set(u.chats.map((c) => c.id)), + images: new Set(u.images.map((i) => i.id)), + items: new Set(u.artifacts.items.map((i) => i.id)), + versions: new Set(u.artifacts.versions.map((v) => v.key)) + } + const merged: Staged = earlier + ? { + session: { + ...u.session, + lastSeenCount: Math.max( + earlier.session.lastSeenCount ?? 0, + u.session.lastSeenCount ?? 0 + ) + }, + sync: { + ...u.sync, + chats: { ...earlier.sync.chats, ...u.sync.chats }, + images: { ...earlier.sync.images, ...u.sync.images }, + artifacts: b.artifacts !== undefined ? u.sync.artifacts : earlier.sync.artifacts + }, + pieces: union(earlier.pieces, written) + } + : { session: u.session, sync: u.sync, pieces: written } + merged.listing = b.listing + // The staging row goes before the pieces, and outlives a restore cut short: the + // next one reads it to know what to delete. The record replaces it. + const stagingPieces = union(earlierStaging.get(b.id) ?? noPieces(), merged.pieces) + await writeSync( + [ + { + id: b.id, + ws, + head: '', + chats: {}, + images: {}, + staging: { + chats: [...stagingPieces.chats], + images: [...stagingPieces.images], + items: [...stagingPieces.items], + versions: [...stagingPieces.versions] + } + } + ], + email + ) + // Written over whatever is there: the session is absent locally, so its pieces + // can only be what an earlier restore staged before it was cut short, and the + // backup may have moved on since. + try { + if (!(await importArtifacts(u.artifacts.items, u.artifacts.versions, email, true))) continue + if (!(await importStoredChats(u.chats, u.images, email, true))) continue + } catch (e) { + console.error(`Could not restore session ${u.session.id}`, e) + continue + } + if (b.next) { + staged.set(b.id, merged) + resumes.push(b.next) + continue + } + ready.push(merged) + } + if (ready.length === 0) continue + // The staged pieces the backup no longer has go before the record: once the record + // is there, no restore looks at the session again, and a flush would push them back. + // A prune that could not run leaves the session, its pieces and its staging row for + // the next restore. + const pruned: Staged[] = [] + for (const r of ready) { + const prior = earlierStaging.get(r.session.id) + if (prior) { + const gone = (was: Set, now: Set) => + new Set([...was].filter((id) => !now.has(id))) + const ok = + (await pruneSessionChats( + r.session.id, + gone(prior.chats, r.pieces.chats), + gone(prior.images, r.pieces.images), + email + )) && + (await pruneSessionArtifacts( + r.session.id, + gone(prior.items, r.pieces.items), + gone(prior.versions, r.pieces.versions), + email + )) + if (!ok) { + console.warn(`Session backup ${r.session.id} could not be tidied; left for later`) + continue + } + earlierStaging.delete(r.session.id) + } + pruned.push(r) + } + toImport.push(...pruned) + } + // The records land together once every pull is done, so the family is listed again + // once per workspace rather than per answer; the pieces are in place either way, and a + // restore cut short before this leaves them staged for the next. + if (toImport.length === 0) return + const superseded = await verify(toImport.map((r) => r.session.id)) + const current = toImport.filter((r) => !superseded.has(r.session.id)) + if (current.length === 0) return + const imported = new Set( + await importSessions( + current.map((r) => r.session), + email + ) + ) + await writeSync( + current.filter((r) => imported.has(r.session.id)).map((r) => r.sync), + email + ) + if (imported.size > 0) logFeatureUsage('ai_session', 'restored', { value: imported.size }) +} + +/** + * Restore the sessions of the workspace family the user is looking at (the workspace and + * its forks), once per workspace per page load. Sessions that exist here are never + * touched; only ones this browser lacks are brought back. + */ +export function restoreSessionBackups(currentWorkspace: string): void { + if (!BROWSER) return + // A restore writes an absent session's pieces page by page and prunes what an earlier + // one staged: two tabs doing that at once would write over each other, so it runs only + // under the tab lock. Where Web Locks do not exist (a plain http origin), the browser + // still backs up; its sessions come back on a secure one. + if (!hasWebLocks()) return + const email = getCurrentUserEmail() + if (!email) return + const all = get(userWorkspaces) + const root = workspaceRootId(currentWorkspace, all) ?? currentWorkspace + const family = new Set([currentWorkspace]) + for (const w of all) if ((workspaceRootId(w.id, all) ?? w.id) === root) family.add(w.id) + const todo = [...family].filter((ws) => !restoredWorkspaces.has(ws) && !isOff(ws)) + for (const ws of todo) restoredWorkspaces.add(ws) + if (todo.length > 0) { + void enqueue(() => withUserLock(email, () => restoreFamily([...family], todo, email), true)) + } +} + +/** The workspace's backups were just turned on or off from this page: what was learnt of + * them is forgotten, and the next flush and a restore find out afresh. */ +export function backupSettingsChanged(ws: string): void { + forgetOff(ws) + wsState.delete(ws) + restoredWorkspaces.delete(ws) + // The rows went stale when the backups went off: the sessions are marked again. + backfilled = false + scheduleFlush() + restoreSessionBackups(ws) +} + +// --- Wiring --- + +if (BROWSER) { + onMirrorSignal((signal) => { + // A mark for another user waits for that user's next load. + const mine = !signal.email || signal.email === getCurrentUserEmail() + if (signal.kind === 'dirty') { + if (bumpDirty(signal.sessionId, signal.email)) { + if (mine) scheduleFlush() + } else { + if (mine && !unwritableMarks.has(signal.sessionId)) { + unwritableMarks.set(signal.sessionId, 0) + } + void bumpViaSyncRow(signal.sessionId, signal.email).finally(() => { + if (mine) scheduleFlush() + }) + } + return + } + if (!addRemoved(signal.sessionId, signal.workspaceId, true, signal.email)) { + void removeViaSyncRow(signal.sessionId, signal.workspaceId, signal.email) + } + if (mine) scheduleFlush() + }) + onUserChange((email) => { + clearTimers() + clearTimeout(retryTimer) + retryAt = 0 + retryMs = RETRY_MIN_MS + wsState.clear() + clearOffTimers() + restoredWorkspaces.clear() + backfilled = false + unwritableMarks.clear() + if (email) setTimeout(runFlush, STARTUP_DELAY_MS) + }) + // A tab going to the background may not come back: carry what it has now. + if (typeof document !== 'undefined') { + document.addEventListener('visibilitychange', () => { + if (document.visibilityState === 'hidden') runFlush() + }) + } +} + +/** Test-only: run a flush now, outside the timers. */ +export function __flushForTesting(): Promise { + return enqueue(flush) +} + +/** Test-only: what the sync table holds for the user. */ +export async function __syncRowsForTesting(email: string): Promise { + return allSyncRows(email) +} + +/** Test-only: how long a workspace found off is left alone before the page asks again. */ +export function __setOffRetryForTesting(ms: number): void { + offRetryMs = ms +} + +/** Test-only: plant sync rows, as an earlier page load would have left them. */ +export function __writeSyncForTesting(rows: MirrorSyncState[], email: string): Promise { + return writeSync(rows, email) +} + +/** Test-only: wait for whatever flush or restore is queued. */ +export function __settleForTesting(): Promise { + return enqueue(async () => {}) +} + +/** Test-only: forget every page-lifetime decision, and let go of the sync store so the + * next open lands in the test's fresh IndexedDB rather than the cached connection. */ +export function __resetMirrorForTesting(): void { + syncDbh.close() + clearTimers() + clearTimeout(retryTimer) + retryAt = 0 + retryMs = RETRY_MIN_MS + wsState.clear() + clearOffTimers() + offRetryMs = OFF_RETRY_MS + restoredWorkspaces.clear() + backfilled = false + unwritableMarks.clear() +} diff --git a/frontend/src/lib/components/sessions/sessionMirror.test.ts b/frontend/src/lib/components/sessions/sessionMirror.test.ts new file mode 100644 index 0000000000..d7260c0878 --- /dev/null +++ b/frontend/src/lib/components/sessions/sessionMirror.test.ts @@ -0,0 +1,1449 @@ +import { describe, it, expect, beforeEach, vi } from 'vitest' +import { IDBFactory } from 'fake-indexeddb' + +// The stores are BROWSER-gated; the vitest "server" env reports false. +vi.mock('esm-env', async (importOriginal) => ({ + ...(await importOriginal()), + BROWSER: true +})) + +const { pushMock, listMock, pullMock } = vi.hoisted(() => ({ + pushMock: vi.fn(), + listMock: vi.fn(), + pullMock: vi.fn() +})) +vi.mock('$lib/gen', async (orig) => { + const actual = await orig() + return { + ...actual, + AiService: { + ...actual.AiService, + pushAiSessionBackups: pushMock, + listAiSessionBackups: listMock, + pullAiSessionBackups: pullMock + }, + WorkspaceService: { + ...actual.WorkspaceService, + listUserWorkspaces: vi.fn().mockResolvedValue([]), + getSessionWorkspaceStatus: vi.fn().mockResolvedValue({}) + } + } +}) + +// The chat store during a restore: real, or unreachable for one session. +const { chatImport } = vi.hoisted(() => ({ + chatImport: { unavailable: false, pruneFails: false } +})) +vi.mock('../copilot/chat/HistoryManager.svelte', async (orig) => { + const actual = await orig() + return { + ...actual, + importStoredChats: (...args: Parameters) => + chatImport.unavailable ? Promise.resolve(false) : actual.importStoredChats(...args), + pruneSessionChats: (...args: Parameters) => + chatImport.pruneFails ? Promise.resolve(false) : actual.pruneSessionChats(...args) + } +}) + +/** The Web Locks API, which the node test environment lacks: one holder per name at a + * time, `ifAvailable` answering null while a holder is there. */ +function fakeLockManager(): LockManager { + const tails = new Map>() + return { + request: async ( + name: string, + options: LockOptions | undefined, + cb: (lock: Lock | null) => unknown + ) => { + const prev = tails.get(name) + if (options?.ifAvailable && prev) return cb(null) + const run = (prev ?? Promise.resolve()).then(() => cb({ name, mode: 'exclusive' })) + const tail = run.catch(() => {}) + tails.set(name, tail) + try { + return await run + } finally { + if (tails.get(name) === tail) tails.delete(name) + } + } + } as unknown as LockManager +} + +function setWebLocks(locks: LockManager | undefined): void { + if (typeof navigator === 'undefined') { + Object.defineProperty(globalThis, 'navigator', { + value: {}, + configurable: true, + writable: true + }) + } + Object.defineProperty(navigator, 'locks', { value: locks, configurable: true }) +} + +import { superadmin, userStore, usersWorkspaceStore, type UserExt } from '$lib/stores' +import HistoryManager, { + __resetBackupStoreForTesting, + __resetLegacyChatClaimForTesting, + readStoredChat +} from '../copilot/chat/HistoryManager.svelte' +import { + deleteSession, + importSessions, + putSession, + sessionState, + type Session +} from './sessionState.svelte' +import { markSessionDirty } from './sessionMirrorSignal' +import { + __flushForTesting, + __resetMirrorForTesting, + __settleForTesting, + __setOffRetryForTesting, + __syncRowsForTesting, + __writeSyncForTesting, + backupSettingsChanged, + restoreSessionBackups +} from './sessionMirror.svelte' + +const EMAIL = 'mirror@x.com' +const IMAGE = 'data:image/png;base64,AAAA' +const PENDING_PREFIX = `windmill_sessions_mirror_pending::${EMAIL}::` + +function asUser(email: string): UserExt { + return { email, username: email.split('@')[0] } as unknown as UserExt +} +const flush = () => new Promise((r) => setTimeout(r, 0)) + +function pendingKeys(): string[] { + const keys: string[] = [] + for (let i = 0; i < localStorage.length; i++) { + const key = localStorage.key(i) + if (key?.startsWith(PENDING_PREFIX)) keys.push(key.slice(PENDING_PREFIX.length)) + } + return keys.sort() +} + +function removalKeys(): string[] { + return pendingKeys().filter((k) => k.startsWith('r::')) +} + +/** Sessions whose dirty counter no push has covered yet. */ +async function pendingDirty(): Promise { + const flushed = new Map() + for (const r of await __syncRowsForTesting(EMAIL)) { + flushed.set(r.id, r.stale ? -1 : (r.flushedV ?? -1) - (r.extraV ?? 0)) + } + const out: string[] = [] + for (const key of pendingKeys()) { + if (!key.startsWith('d::')) continue + const id = key.slice(3) + if (Number(localStorage.getItem(PENDING_PREFIX + key)) > (flushed.get(id) ?? -1)) out.push(id) + } + return out.sort() +} + +beforeEach(async () => { + ;(globalThis as any).indexedDB = new IDBFactory() + localStorage.clear() + __resetLegacyChatClaimForTesting() + __resetBackupStoreForTesting() + __resetMirrorForTesting() + pushMock.mockReset() + listMock.mockReset() + pullMock.mockReset() + chatImport.unavailable = false + chatImport.pruneFails = false + setWebLocks(fakeLockManager()) + superadmin.set(false) + usersWorkspaceStore.set(undefined) + userStore.set(undefined) + await flush() + sessionState.sessions = [] + userStore.set(asUser(EMAIL)) + await vi.waitFor(() => expect(sessionState.hydrated).toBe(true)) +}) + +// The whole loop, end to end against the real stores: local writes mark, a flush sends +// one batch carrying the record, the chat and its image, reading the session sends +// nothing, and a user delete removes the backup. +describe('sessionMirror flush', () => { + it('pushes what changed, once, and removes a deleted session', async () => { + pushMock.mockResolvedValue({ enabled: true, results: [{ id: 's1' }] }) + const s: Session = { + id: 's1', + name: 'session-1', + createdAt: 1, + workspace_id: 'ws', + chatId: 'c1' + } + sessionState.sessions = [s] + await putSession(s) + + const hm = new HistoryManager() + await hm.init() + hm.setSessionId('s1') + hm.setCurrentChatId('c1') + await hm.saveChat( + [{ role: 'user', content: 'hello', images: [{ dataUrl: IMAGE, name: 'a.png' }] } as never], + [{ role: 'user', content: [{ type: 'image_url', image_url: { url: IMAGE } }] } as never] + ) + + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(1) + const body = pushMock.mock.calls[0][0].requestBody + expect(pushMock.mock.calls[0][0].workspace).toBe('ws') + expect(body.owner).toBe(EMAIL) + expect(body.removed).toBeUndefined() + const [imageEntry, entry] = body.sessions + expect(imageEntry.images).toEqual([{ chat_id: 'c1', id: expect.any(String), data_url: IMAGE }]) + // A first push goes whole, under one token on every part: the head opens it on the + // first part, whichever that is. + expect(imageEntry.whole).toBe(true) + expect(imageEntry.partial).toBe(true) + expect(typeof imageEntry.push).toBe('string') + expect(imageEntry.opens).toBe(true) + expect(imageEntry.head).toEqual({ id: 's1', createdAt: 1, workspace_id: 'ws', chatId: 'c1' }) + expect(entry.head).toBeUndefined() + expect(entry.whole).toBe(true) + expect(entry.push).toBe(imageEntry.push) + expect(entry.opens).toBeUndefined() + expect(entry.chats.map((c: { id: string }) => c.id)).toEqual(['c1']) + // The record keeps its blob ref; bytes travel as the image object only. + expect(JSON.stringify(entry.chats[0].record)).not.toContain(IMAGE) + expect(entry.artifacts).toBeUndefined() + expect(await pendingDirty()).toEqual([]) + + // Reading the session is not a change the backup keeps. + await putSession({ ...s, lastSeenCount: 2, lastActivityAt: 99 }) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(1) + + // A user delete takes the backup with it. + deleteSession('s1') + await flush() + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(2) + expect(pushMock.mock.calls[1][0].requestBody).toEqual({ + owner: EMAIL, + sessions: [], + removed: ['s1'] + }) + expect(pendingKeys()).toEqual([]) + hm.close() + }) + + it('carries a user delete on the sync row when its localStorage mark cannot be written', async () => { + pushMock.mockResolvedValue({ enabled: true, results: [{ id: 'sd' }] }) + const s: Session = { id: 'sd', name: 'session-1', createdAt: 1, workspace_id: 'ws' } + sessionState.sessions = [s] + await putSession(s) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(1) + + // Storage full at the moment of the delete. + const setItem = localStorage.setItem.bind(localStorage) + localStorage.setItem = (key: string, value: string) => { + if (key.includes('::r::')) throw new Error('QuotaExceededError') + setItem(key, value) + } + try { + deleteSession('sd') + } finally { + localStorage.setItem = setItem + } + await flush() + expect(removalKeys()).toEqual([]) + await vi.waitFor(async () => + expect((await __syncRowsForTesting(EMAIL)).find((r) => r.id === 'sd')?.removed).toBe(true) + ) + + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(2) + expect(pushMock.mock.calls[1][0].requestBody).toEqual({ + owner: EMAIL, + sessions: [], + removed: ['sd'] + }) + expect((await __syncRowsForTesting(EMAIL)).some((r) => r.id === 'sd')).toBe(false) + await __settleForTesting() + }) + + it('keeps a delete filed on the sync row while the first push is still in flight', async () => { + const s: Session = { id: 'sr', name: 'session-1', createdAt: 1, workspace_id: 'ws' } + sessionState.sessions = [s] + await putSession(s) + let release!: (value: unknown) => void + pushMock.mockImplementationOnce(() => new Promise((r) => (release = r))) + const inFlight = __flushForTesting() + await vi.waitFor(() => expect(pushMock).toHaveBeenCalledTimes(1)) + + // Storage full at the moment of the delete, the push not yet answered. + const setItem = localStorage.setItem.bind(localStorage) + localStorage.setItem = (key: string, value: string) => { + if (key.includes('::r::')) throw new Error('QuotaExceededError') + setItem(key, value) + } + try { + deleteSession('sr') + } finally { + localStorage.setItem = setItem + } + await vi.waitFor(async () => + expect((await __syncRowsForTesting(EMAIL)).find((r) => r.id === 'sr')?.removed).toBe(true) + ) + release({ enabled: true, results: [{ id: 'sr' }] }) + await inFlight + // The push's own row write did not lose the removal. + expect((await __syncRowsForTesting(EMAIL)).find((r) => r.id === 'sr')?.removed).toBe(true) + + pushMock.mockResolvedValueOnce({ enabled: true, results: [{ id: 'sr' }] }) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(2) + expect(pushMock.mock.calls[1][0].requestBody.removed).toEqual(['sr']) + expect((await __syncRowsForTesting(EMAIL)).some((r) => r.id === 'sr')).toBe(false) + }) + + it('keeps the marks when the push fails, and stops for a workspace without storage', async () => { + const s: Session = { id: 's2', name: 'session-2', createdAt: 1, workspace_id: 'ws' } + const never: Session = { id: 's2b', name: 'session-3', createdAt: 2, workspace_id: 'ws' } + sessionState.sessions = [s, never] + await putSession(s) + + pushMock.mockRejectedValueOnce(new TypeError('network')) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(1) + expect(await pendingDirty()).toEqual(['s2']) + // Still marked: the retry carries it again once the backoff lapses. + __resetMirrorForTesting() + pushMock.mockResolvedValueOnce({ enabled: true, results: [{ id: 's2' }] }) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(2) + expect(await pendingDirty()).toEqual([]) + + // The storage goes away: no further request for the page. A delete keeps its + // removal mark only for a session that was backed up (s2), for when backups are on + // again, or it would come back from the bucket; one never backed up has nothing + // there, so its mark goes rather than piling up on a storage-less instance. + await putSession({ ...s, summary: 'changed' }) + await putSession(never) + __setOffRetryForTesting(50) + pushMock.mockResolvedValueOnce({ enabled: false, results: [] }) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(3) + // The reconcile that runs after login may have re-read the list from the store + // before `never` was in it; deleteSession only acts on sessions it can see. + sessionState.sessions = [s, never] + deleteSession('s2') + deleteSession('s2b') + await flush() + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(3) + expect(pendingKeys()).toEqual(['r::s2::ws']) + + // Backups an admin turns on again elsewhere are noticed once the page's memory of + // them being off expires, with nothing else prompting it: the removal goes then. + listMock.mockResolvedValue({ enabled: true, sessions: [] }) + pushMock.mockResolvedValueOnce({ enabled: true, results: [{ id: 's2' }] }) + await vi.waitFor(() => expect(pushMock).toHaveBeenCalledTimes(4), { timeout: 3000 }) + expect(pushMock.mock.calls[3][0].requestBody.removed).toEqual(['s2']) + await vi.waitFor(() => expect(listMock).toHaveBeenCalledTimes(1)) + await __settleForTesting() + expect(pendingKeys()).toEqual([]) + }) + + it('backs up after a reload an edit whose bump only the sync row carries', async () => { + // Storage full from the session's first mark on: the first push carries the mark this + // page kept, and the edit landing while it is in flight goes onto the row it writes. + const s: Session = { id: 'sx', name: 'session-1', createdAt: 1, workspace_id: 'ws' } + const setItem = localStorage.setItem.bind(localStorage) + localStorage.setItem = (key: string, value: string) => { + if (key.includes('::d::')) throw new Error('QuotaExceededError') + setItem(key, value) + } + try { + sessionState.sessions = [s] + await putSession(s) + let release!: (value: unknown) => void + pushMock.mockImplementationOnce(() => new Promise((r) => (release = r))) + const inFlight = __flushForTesting() + await vi.waitFor(() => expect(pushMock).toHaveBeenCalledTimes(1)) + await putSession({ ...s, summary: 'second' }) + release({ enabled: true, results: [{ id: 'sx' }] }) + await inFlight + expect(pendingKeys()).toEqual([]) + const row = (await __syncRowsForTesting(EMAIL)).find((r) => r.id === 'sx') + expect((row?.extraV ?? 0) > (row?.flushedV ?? -1)).toBe(true) + + // A reload, storage still full: nothing in localStorage or memory names the + // session, the row does. + __resetMirrorForTesting() + pushMock.mockResolvedValueOnce({ enabled: true, results: [{ id: 'sx' }] }) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(2) + expect(pushMock.mock.calls[1][0].requestBody.sessions[0].head.summary).toBe('second') + expect(await pendingDirty()).toEqual([]) + + // The next refused bump counts from what that push retired (the row carries it; + // `pendingDirty` reads localStorage marks only, so the push is the check). + await putSession({ ...s, summary: 'third' }) + await vi.waitFor(async () => + expect((await __syncRowsForTesting(EMAIL)).find((r) => r.id === 'sx')?.extraV).toBe(3) + ) + pushMock.mockResolvedValueOnce({ enabled: true, results: [{ id: 'sx' }] }) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(3) + expect(pushMock.mock.calls[2][0].requestBody.sessions[0].head.summary).toBe('third') + expect(await pendingDirty()).toEqual([]) + } finally { + localStorage.setItem = setItem + } + }) + + it('backs up and restores again at once when the backups are turned on from this page', async () => { + const s: Session = { id: 'so', name: 'session-1', createdAt: 1, workspace_id: 'ws' } + sessionState.sessions = [s] + await putSession(s) + pushMock.mockResolvedValueOnce({ enabled: true, results: [{ id: 'so' }] }) + await __flushForTesting() + await putSession({ ...s, summary: 'changed' }) + pushMock.mockResolvedValueOnce({ enabled: false, results: [] }) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(2) + await putSession({ ...s, summary: 'changed again' }) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(2) + + listMock.mockResolvedValue({ enabled: true, sessions: [] }) + pushMock.mockResolvedValueOnce({ enabled: true, results: [{ id: 'so' }] }) + usersWorkspaceStore.set({ email: EMAIL, workspaces: [] } as never) + backupSettingsChanged('ws') + await __settleForTesting() + await __flushForTesting() + expect(listMock).toHaveBeenCalledTimes(1) + expect(pushMock).toHaveBeenCalledTimes(3) + // The rows went stale while the backups were off: the session goes whole. + expect(pushMock.mock.calls[2][0].requestBody.sessions[0].whole).toBe(true) + expect(pushMock.mock.calls[2][0].requestBody.sessions[0].head.summary).toBe('changed again') + expect(await pendingDirty()).toEqual([]) + }) + + it('settles nothing of a request that failed, even a session whose parts were still to come', async () => { + // Enough sessions for two requests: the first fails, the second is never sent. + const ids = Array.from({ length: 101 }, (_, i) => `m${i}`) + for (const id of ids) { + await putSession({ id, name: id, createdAt: 1, workspace_id: 'ws' }) + } + pushMock.mockRejectedValueOnce(new TypeError('network')) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(1) + expect(pushMock.mock.calls[0][0].requestBody.sessions).toHaveLength(100) + expect(await pendingDirty()).toHaveLength(101) + + // Once the backoff lapses, every one of them is carried again. + __resetMirrorForTesting() + pushMock.mockImplementation( + async ({ requestBody }: { requestBody: { sessions: { id: string }[] } }) => ({ + enabled: true, + results: requestBody.sessions.map((s) => ({ id: s.id })) + }) + ) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(3) + expect( + pushMock.mock.calls + .slice(1) + .flatMap((c) => c[0].requestBody.sessions.map((s: { id: string }) => s.id)) + .sort() + ).toEqual([...ids].sort()) + expect(await pendingDirty()).toEqual([]) + }) + + it('backs off when the server could not store a session, keeping its mark', async () => { + const s: Session = { id: 's3', name: 'session-3', createdAt: 1, workspace_id: 'ws' } + sessionState.sessions = [s] + await putSession(s) + + pushMock.mockResolvedValue({ enabled: true, results: [{ id: 's3', error: 'bucket refused' }] }) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(1) + // Still marked, but not re-sent until the backoff lapses. + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(1) + expect(await pendingDirty()).toEqual(['s3']) + }) + + it('fails only the sessions of a request the server found too large', async () => { + const ids = Array.from({ length: 101 }, (_, i) => `t${i}`) + for (const id of ids) { + await putSession({ id, name: id, createdAt: 1, workspace_id: 'ws' }) + } + const { ApiError } = await import('$lib/gen') + pushMock + .mockRejectedValueOnce( + new ApiError({ method: 'POST', url: '' } as never, { status: 413 } as never, 'too large') + ) + .mockImplementation( + async ({ requestBody }: { requestBody: { sessions: { id: string }[] } }) => ({ + enabled: true, + results: requestBody.sessions.map((s) => ({ id: s.id })) + }) + ) + await __flushForTesting() + // The second request still went out and settled its session; the first's stay marked. + expect(pushMock).toHaveBeenCalledTimes(2) + expect(await pendingDirty()).toHaveLength(100) + const settled = pushMock.mock.calls[1][0].requestBody.sessions[0].id + expect(await pendingDirty()).not.toContain(settled) + }) + + it('moves a session whole into its new workspace and files the old copy for removal', async () => { + const s: Session = { id: 'mv', name: 'session-1', createdAt: 1, workspace_id: 'ws' } + sessionState.sessions = [s] + await putSession(s) + pushMock.mockResolvedValueOnce({ enabled: true, results: [{ id: 'mv' }] }) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(1) + + // The old workspace's backups go off before the move: its rows go stale, but they + // still say where the copy is. + await putSession({ ...s, summary: 'changed' }) + pushMock.mockResolvedValueOnce({ enabled: false, results: [] }) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(2) + + await putSession({ ...s, summary: 'changed', workspace_id: 'ws2' }) + pushMock.mockResolvedValueOnce({ enabled: true, results: [{ id: 'mv' }] }) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(3) + expect(pushMock.mock.calls[2][0].workspace).toBe('ws2') + expect(pushMock.mock.calls[2][0].requestBody.sessions[0].head.workspace_id).toBe('ws2') + // The removal waits for the old workspace's backups to be on again. + expect(removalKeys()).toEqual(['r::mv::ws']) + expect(await pendingDirty()).toEqual([]) + }) + + it('pushes every session whole again once the server answers from another storage', async () => { + const a: Session = { + id: 'sa', + name: 'session-1', + createdAt: 1, + workspace_id: 'ws', + chatId: 'ca' + } + const b: Session = { id: 'sb', name: 'session-2', createdAt: 2, workspace_id: 'ws' } + sessionState.sessions = [a, b] + await putSession(a) + await putSession(b) + const hm = new HistoryManager() + await hm.init() + hm.setSessionId('sa') + hm.setCurrentChatId('ca') + await hm.saveChat( + [{ role: 'user', content: 'hello' } as never], + [{ role: 'user', content: 'hello' } as never] + ) + pushMock.mockResolvedValueOnce({ + enabled: true, + storage_id: 'bucket-1', + results: [{ id: 'sa' }, { id: 'sb' }] + }) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(1) + + // Only `sa`'s record changes, and the answer names a new storage: it holds `sb` + // nowhere and `sa` only in the part that went, so both are backed up whole again. + await putSession({ ...a, summary: 'changed' }) + pushMock.mockResolvedValueOnce({ + enabled: true, + storage_id: 'bucket-2', + results: [{ id: 'sa' }] + }) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(2) + expect(pushMock.mock.calls[1][0].requestBody.sessions[0].chats).toBeUndefined() + expect(await pendingDirty()).toEqual(['sa', 'sb']) + + pushMock.mockResolvedValueOnce({ + enabled: true, + storage_id: 'bucket-2', + results: [{ id: 'sa' }, { id: 'sb' }] + }) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(3) + const entries = pushMock.mock.calls[2][0].requestBody.sessions + expect(entries.map((s: { id: string }) => s.id).sort()).toEqual(['sa', 'sb']) + expect(entries.every((s: { head?: unknown }) => s.head !== undefined)).toBe(true) + expect(entries.find((s: { id: string }) => s.id === 'sa').chats).toHaveLength(1) + expect(await pendingDirty()).toEqual([]) + }) + + /** A session with three chats of 1.5 MB (record ~3 MB each): its entry splits past the + * 8 MB request target, so it spans two requests. */ + async function splitSession(id: string): Promise { + const s: Session = { id, name: 'session-1', createdAt: 1, workspace_id: 'ws', chatId: 'c1' } + sessionState.sessions = [s] + await putSession(s) + const hm = new HistoryManager() + await hm.init() + hm.setSessionId(id) + const big = 'x'.repeat(1.5 * 1024 * 1024) + for (const cid of ['c1', 'c2', 'c3']) { + hm.setCurrentChatId(cid) + await hm.saveChat( + [{ role: 'user', content: big } as never], + [{ role: 'user', content: big } as never] + ) + } + } + + it('settles nothing of a session whose parts were answered from different storages', async () => { + await splitSession('sp') + pushMock + .mockResolvedValueOnce({ enabled: true, storage_id: 'bucket-1', results: [{ id: 'sp' }] }) + .mockResolvedValueOnce({ enabled: true, storage_id: 'bucket-2', results: [{ id: 'sp' }] }) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(2) + // The first part opens the session whole with its head; only the last completes + // the entry server-side. + const [first, last] = pushMock.mock.calls.map((c) => c[0].requestBody.sessions[0]) + expect(first.partial).toBe(true) + expect(first.whole).toBe(true) + expect(typeof first.push).toBe('string') + expect(first.opens).toBe(true) + expect(first.head).toBeDefined() + expect(last.partial).toBeUndefined() + expect(last.whole).toBe(true) + expect(last.push).toBe(first.push) + expect(last.opens).toBeUndefined() + expect(last.head).toBeUndefined() + expect(await pendingDirty()).toEqual(['sp']) + expect(await __syncRowsForTesting(EMAIL)).toEqual([]) + }) + + it('names an incremental push split over requests on each of its parts', async () => { + await splitSession('si') + pushMock.mockResolvedValue({ enabled: true, results: [{ id: 'si' }] }) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(2) + expect(await pendingDirty()).toEqual([]) + // Every chat changes: the update spans two requests again, incremental this time. + const hm = new HistoryManager() + await hm.init() + hm.setSessionId('si') + const big = 'y'.repeat(1.5 * 1024 * 1024) + for (const cid of ['c1', 'c2', 'c3']) { + hm.setCurrentChatId(cid) + await hm.saveChat( + [{ role: 'user', content: big } as never], + [{ role: 'user', content: big } as never] + ) + } + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(4) + const [first, last] = pushMock.mock.calls.slice(2).map((c) => c[0].requestBody.sessions[0]) + expect(first.whole).toBeUndefined() + expect(first.partial).toBe(true) + expect(typeof first.push).toBe('string') + expect(first.opens).toBe(true) + expect(last.partial).toBeUndefined() + expect(last.push).toBe(first.push) + expect(last.opens).toBeUndefined() + expect(await pendingDirty()).toEqual([]) + hm.close() + }) + + it('retires a removal only once the storage holding the backup answered it', async () => { + const s: Session = { id: 'sr2', name: 'session-1', createdAt: 1, workspace_id: 'ws' } + sessionState.sessions = [s] + await putSession(s) + pushMock.mockResolvedValueOnce({ enabled: true, storage_id: 'A', results: [{ id: 'sr2' }] }) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(1) + + // The workspace moved to another storage before the delete: the copy in A stays. + deleteSession('sr2') + await flush() + pushMock.mockResolvedValueOnce({ enabled: true, storage_id: 'B', results: [{ id: 'sr2' }] }) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(2) + expect(removalKeys()).toEqual(['r::sr2::ws']) + expect((await __syncRowsForTesting(EMAIL)).some((r) => r.id === 'sr2')).toBe(true) + + // Back on A, the removal lands. + pushMock.mockResolvedValueOnce({ enabled: true, storage_id: 'A', results: [{ id: 'sr2' }] }) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(3) + expect(pushMock.mock.calls[2][0].requestBody.removed).toEqual(['sr2']) + expect(removalKeys()).toEqual([]) + expect((await __syncRowsForTesting(EMAIL)).some((r) => r.id === 'sr2')).toBe(false) + }) + + it('removes a deleted session from every storage that holds a copy of it', async () => { + const s: Session = { id: 'sr3', name: 'session-1', createdAt: 1, workspace_id: 'ws' } + sessionState.sessions = [s] + await putSession(s) + pushMock.mockResolvedValueOnce({ enabled: true, storage_id: 'A', results: [{ id: 'sr3' }] }) + await __flushForTesting() + // The workspace moves to B: the row goes stale, the session goes whole to B and + // settles there, and the row remembers the copy A keeps. + await putSession({ ...s, summary: 'changed' }) + pushMock.mockResolvedValue({ enabled: true, storage_id: 'B', results: [{ id: 'sr3' }] }) + await __flushForTesting() + await __flushForTesting() + expect(await pendingDirty()).toEqual([]) + const row = (await __syncRowsForTesting(EMAIL)).find((r) => r.id === 'sr3') + expect(row?.storageId).toBe('B') + expect(row?.alsoIn).toEqual(['A']) + + // Deleted while on B: B's copy goes, and the mark waits for A to answer. + deleteSession('sr3') + await flush() + await __flushForTesting() + expect(removalKeys()).toEqual(['r::sr3::ws']) + const narrowed = (await __syncRowsForTesting(EMAIL)).find((r) => r.id === 'sr3') + expect(narrowed?.storageId).toBe('A') + expect(narrowed?.alsoIn).toBeUndefined() + + // Back on A, the copy there goes too, and only then is the removal done. + pushMock.mockResolvedValue({ enabled: true, storage_id: 'A', results: [{ id: 'sr3' }] }) + await __flushForTesting() + expect(pushMock.mock.lastCall?.[0].requestBody.removed).toEqual(['sr3']) + expect(removalKeys()).toEqual([]) + expect((await __syncRowsForTesting(EMAIL)).some((r) => r.id === 'sr3')).toBe(false) + }) + + it("removes a moved session's old copy from the storage that held it, whatever its old workspace is on now", async () => { + const s: Session = { id: 'mv2', name: 'session-1', createdAt: 1, workspace_id: 'ws' } + sessionState.sessions = [s] + await putSession(s) + pushMock.mockResolvedValueOnce({ enabled: true, storage_id: 'A', results: [{ id: 'mv2' }] }) + await __flushForTesting() + // The old workspace's backups go off, then the session moves: the new workspace's + // row replaces the old one, so the mark carries where the old copy is. + await putSession({ ...s, summary: 'changed' }) + pushMock.mockResolvedValueOnce({ enabled: false, results: [] }) + await __flushForTesting() + await putSession({ ...s, summary: 'changed', workspace_id: 'ws2' }) + pushMock.mockResolvedValueOnce({ enabled: true, storage_id: 'B', results: [{ id: 'mv2' }] }) + await __flushForTesting() + expect(removalKeys()).toEqual(['r::mv2::ws']) + + // On the next page load the old workspace is on again, but on another storage: the + // removal there deletes nothing, and the mark waits for the storage holding the copy. + __resetMirrorForTesting() + pushMock.mockResolvedValueOnce({ enabled: true, storage_id: 'C', results: [{ id: 'mv2' }] }) + await __flushForTesting() + expect(pushMock.mock.lastCall?.[0].requestBody.removed).toEqual(['mv2']) + expect(removalKeys()).toEqual(['r::mv2::ws']) + pushMock.mockResolvedValueOnce({ enabled: true, storage_id: 'A', results: [{ id: 'mv2' }] }) + await __flushForTesting() + expect(pushMock.mock.lastCall?.[0].requestBody.removed).toEqual(['mv2']) + expect(removalKeys()).toEqual([]) + expect((await __syncRowsForTesting(EMAIL)).find((r) => r.id === 'mv2')?.ws).toBe('ws2') + }) + + it('takes a bumped backup generation as a new storage for the rows, not for a removal', async () => { + const a: Session = { id: 'ga', name: 'session-1', createdAt: 1, workspace_id: 'ws' } + const b: Session = { id: 'gb', name: 'session-2', createdAt: 2, workspace_id: 'ws' } + sessionState.sessions = [a, b] + await putSession(a) + await putSession(b) + pushMock.mockResolvedValueOnce({ + enabled: true, + storage_id: 'A', + backup_generation: 0, + results: [{ id: 'ga' }, { id: 'gb' }] + }) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(1) + + // The key was rotated before the delete: the removal is done (the old generation is + // gone with the rotation), and `gb` goes whole again under the new one. + deleteSession('ga') + await flush() + pushMock.mockResolvedValueOnce({ + enabled: true, + storage_id: 'A', + backup_generation: 1, + results: [{ id: 'ga' }] + }) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(2) + expect(pushMock.mock.calls[1][0].requestBody.removed).toEqual(['ga']) + expect(removalKeys()).toEqual([]) + expect(await pendingDirty()).toEqual(['gb']) + }) + + it('sends a session whole again once the server says its head is gone', async () => { + const s: Session = { + id: 'sh', + name: 'session-1', + createdAt: 1, + workspace_id: 'ws', + chatId: 'c1' + } + sessionState.sessions = [s] + await putSession(s) + const hm = new HistoryManager() + await hm.init() + hm.setSessionId('sh') + hm.setCurrentChatId('c1') + await hm.saveChat( + [{ role: 'user', content: 'a' } as never], + [{ role: 'user', content: 'a' } as never] + ) + pushMock.mockResolvedValueOnce({ enabled: true, results: [{ id: 'sh' }] }) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(1) + + // Another device removed the backup; the next chat-only push finds no head there. + hm.setCurrentChatId('c2') + await hm.saveChat( + [{ role: 'user', content: 'ab' } as never], + [{ role: 'user', content: 'ab' } as never] + ) + pushMock.mockResolvedValueOnce({ enabled: true, results: [{ id: 'sh', needs_whole: true }] }) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(2) + expect(pushMock.mock.calls[1][0].requestBody.sessions[0].whole).toBeUndefined() + expect(await pendingDirty()).toEqual(['sh']) + pushMock.mockResolvedValueOnce({ enabled: true, results: [{ id: 'sh' }] }) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(3) + const whole = pushMock.mock.calls[2][0].requestBody.sessions[0] + expect(whole.whole).toBe(true) + expect(whole.push).toBeUndefined() + expect(whole.head).toBeDefined() + expect(whole.chats.map((c: { id: string }) => c.id).sort()).toEqual(['c1', 'c2']) + expect(await pendingDirty()).toEqual([]) + hm.close() + }) + + it('holds the rest of a session back once the server refused a part of it', async () => { + await splitSession('sf') + pushMock.mockResolvedValueOnce({ enabled: true, results: [{ id: 'sf', error: 'boom' }] }) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(1) + expect(await pendingDirty()).toEqual(['sf']) + expect(await __syncRowsForTesting(EMAIL)).toEqual([]) + }) + + it('keeps an edit whose dirty mark cannot be written while the session has no row yet', async () => { + const s: Session = { id: 'sn', name: 'session-1', createdAt: 1, workspace_id: 'ws' } + sessionState.sessions = [s] + await putSession(s) + // The first push is held open; storage is full for the edit that lands meanwhile. + let release!: (value: unknown) => void + pushMock.mockImplementationOnce(() => new Promise((r) => (release = r))) + const inFlight = __flushForTesting() + await vi.waitFor(() => expect(pushMock).toHaveBeenCalledTimes(1)) + const setItem = localStorage.setItem.bind(localStorage) + localStorage.setItem = (key: string, value: string) => { + if (key.includes('::d::')) throw new Error('QuotaExceededError') + setItem(key, value) + } + try { + await putSession({ ...s, summary: 'second' }) + } finally { + localStorage.setItem = setItem + } + release({ enabled: true, results: [{ id: 'sn' }] }) + await inFlight + // The row the push wrote took the bump over, so the edit is still pending. + expect((await __syncRowsForTesting(EMAIL)).find((r) => r.id === 'sn')?.extraV).toBe(1) + expect(await pendingDirty()).toEqual(['sn']) + pushMock.mockResolvedValueOnce({ enabled: true, results: [{ id: 'sn' }] }) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(2) + expect(pushMock.mock.calls[1][0].requestBody.sessions[0].head.summary).toBe('second') + expect(await pendingDirty()).toEqual([]) + }) + + it('carries an edit on the sync row when its dirty mark cannot be written, even during a push', async () => { + pushMock.mockResolvedValueOnce({ enabled: true, results: [{ id: 'sm' }] }) + const s: Session = { id: 'sm', name: 'session-1', createdAt: 1, workspace_id: 'ws' } + sessionState.sessions = [s] + await putSession(s) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(1) + + // An edit's push is held open; storage is full for the edit that follows. + let release!: (value: unknown) => void + pushMock.mockImplementationOnce(() => new Promise((r) => (release = r))) + await putSession({ ...s, summary: 'first' }) + const inFlight = __flushForTesting() + await vi.waitFor(() => expect(pushMock).toHaveBeenCalledTimes(2)) + const setItem = localStorage.setItem.bind(localStorage) + localStorage.setItem = (key: string, value: string) => { + if (key.includes('::d::')) throw new Error('QuotaExceededError') + setItem(key, value) + } + try { + await putSession({ ...s, summary: 'second' }) + } finally { + localStorage.setItem = setItem + } + await vi.waitFor(async () => + expect((await __syncRowsForTesting(EMAIL)).find((r) => r.id === 'sm')?.extraV).toBe(1) + ) + release({ enabled: true, results: [{ id: 'sm' }] }) + await inFlight + // The push's own row write kept the bump, so the later edit is still pending. + expect(await pendingDirty()).toEqual(['sm']) + pushMock.mockResolvedValueOnce({ enabled: true, results: [{ id: 'sm' }] }) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(3) + expect(pushMock.mock.calls[2][0].requestBody.sessions[0].head.summary).toBe('second') + expect(await pendingDirty()).toEqual([]) + }) + + it('files a mark under the user whose store the write landed in', async () => { + markSessionDirty('sw', undefined, 'other@x.com') + expect(localStorage.getItem('windmill_sessions_mirror_pending::other@x.com::d::sw')).toBe('1') + expect(pendingKeys()).toEqual([]) + + // The mark of another user that cannot be written goes to that user's own rows. + const { openDB } = await import('idb') + const theirs = await openDB('windmill-sessions-mirror::other@x.com', 1, { + upgrade: (db) => db.createObjectStore('sync', { keyPath: 'id' }) + }) + await theirs.put('sync', { id: 'so', ws: 'ws', head: 'h', chats: {}, images: {}, flushedV: 1 }) + theirs.close() + const setItem = localStorage.setItem.bind(localStorage) + localStorage.setItem = (key: string, value: string) => { + if (key.includes('::d::')) throw new Error('QuotaExceededError') + setItem(key, value) + } + try { + markSessionDirty('so', undefined, 'other@x.com') + } finally { + localStorage.setItem = setItem + } + await vi.waitFor(async () => { + const db = await openDB('windmill-sessions-mirror::other@x.com', 1) + try { + expect((await db.get('sync', 'so'))?.extraV).toBe(1) + } finally { + db.close() + } + }) + }) + + it('keeps the marks when the server refuses a request, for the next page load', async () => { + const s: Session = { id: 's4', name: 'session-4', createdAt: 1, workspace_id: 'ws' } + sessionState.sessions = [s] + await putSession(s) + + const { ApiError } = await import('$lib/gen') + pushMock.mockRejectedValue( + new ApiError({ method: 'POST', url: '' } as never, { status: 400 } as never, 'quota') + ) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(1) + // Nothing more for this page, marks untouched by the follow-up flush. + await putSession({ ...s, summary: 'changed' }) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(1) + expect(await pendingDirty()).toEqual(['s4']) + }) +}) + +describe('sessionMirror restore', () => { + const backup = { + id: 's9', + head: { id: 's9', workspace_id: 'ws', createdAt: 5, chatId: 'c9', summary: 'remote' }, + chats: [ + { + id: 'c9', + record: { + id: 'c9', + sessionId: 's9', + title: 't', + lastModified: 7, + actualMessages: [], + displayMessages: [{ role: 'user', content: 'hi' }] + } + } + ], + images: [] + } + + it('brings back a session the browser lacks, and records nothing for one whose chats could not be written', async () => { + listMock.mockResolvedValue({ + enabled: true, + sessions: [{ id: 's9', updated_at: '2026-09-14T00:00:00Z' }] + }) + pullMock.mockResolvedValue({ enabled: true, sessions: [backup], deferred: [] }) + usersWorkspaceStore.set({ email: EMAIL, workspaces: [] } as never) + + // A removal pending for another workspace (the session moved here from it) does + // not stand in the way of restoring this workspace's copy. + localStorage.setItem(`${PENDING_PREFIX}r::s9::elsewhere`, '1') + chatImport.unavailable = true + restoreSessionBackups('ws') + await __settleForTesting() + expect(pullMock).toHaveBeenCalledTimes(1) + expect(sessionState.sessions.map((s) => s.id)).toEqual([]) + // Not recorded as restored: the next restore tries again, and no flush can push a + // transcript-less copy over the backup. + __resetMirrorForTesting() + chatImport.unavailable = false + restoreSessionBackups('ws') + await __settleForTesting() + expect(pullMock).toHaveBeenCalledTimes(2) + await vi.waitFor(() => expect(sessionState.sessions.map((s) => s.id)).toEqual(['s9'])) + const restored = sessionState.sessions[0] + expect(restored.name).toBe('session-1') + expect(restored.summary).toBe('remote') + expect(restored.lastSeenCount).toBe(1) + expect((await readStoredChat('c9', EMAIL))?.displayMessages).toHaveLength(1) + + // Restored state is what the backup holds: nothing to push (the other workspace's + // removal is its own request, not part of this check). + localStorage.removeItem(`${PENDING_PREFIX}r::s9::elsewhere`) + await __flushForTesting() + expect(pushMock).not.toHaveBeenCalled() + }) + + it('imports a session that came in pages only once the last page arrived', async () => { + listMock.mockResolvedValue({ + enabled: true, + sessions: [{ id: 's9', updated_at: '2026-09-14T00:00:00Z' }] + }) + const cursor = { id: 's9', images: false, after: 'sessions/s9/chats/c9.json' } + const c9b = { ...backup.chats[0], id: 'c9b', record: { ...backup.chats[0].record, id: 'c9b' } } + pullMock + .mockResolvedValueOnce({ + enabled: true, + sessions: [{ ...backup, next: cursor }], + deferred: [] + }) + .mockResolvedValueOnce({ + enabled: true, + sessions: [{ ...backup, chats: [c9b] }], + deferred: [] + }) + usersWorkspaceStore.set({ email: EMAIL, workspaces: [] } as never) + restoreSessionBackups('ws') + await __settleForTesting() + expect(pullMock).toHaveBeenCalledTimes(2) + expect(pullMock.mock.calls[1][0].requestBody).toEqual({ ids: ['s9'], resume: cursor }) + await vi.waitFor(() => expect(sessionState.sessions.map((s) => s.id)).toEqual(['s9'])) + expect((await readStoredChat('c9', EMAIL))?.displayMessages).toHaveLength(1) + expect((await readStoredChat('c9b', EMAIL))?.id).toBe('c9b') + }) + + it('starts a session over when its only page was read while the backup moved', async () => { + listMock.mockResolvedValue({ + enabled: true, + sessions: [{ id: 's9', updated_at: '2026-09-14T00:00:00Z' }] + }) + const c9b = { ...backup.chats[0], id: 'c9b', record: { ...backup.chats[0].record, id: 'c9b' } } + pullMock + .mockResolvedValueOnce({ + enabled: true, + sessions: [{ ...backup, chats: [backup.chats[0]], moved: true }], + deferred: [] + }) + .mockResolvedValueOnce({ + enabled: true, + sessions: [{ ...backup, chats: [backup.chats[0], c9b] }], + deferred: [] + }) + usersWorkspaceStore.set({ email: EMAIL, workspaces: [] } as never) + restoreSessionBackups('ws') + await __settleForTesting() + expect(pullMock).toHaveBeenCalledTimes(2) + await vi.waitFor(() => expect(sessionState.sessions.map((s) => s.id)).toEqual(['s9'])) + expect((await readStoredChat('c9b', EMAIL))?.id).toBe('c9b') + }) + + it('brings a session two workspaces of the family list back from the copy that moved last', async () => { + // Both copies carry the same modification time (a store reports them coarsely): the + // move count tells them apart. + listMock.mockImplementation(async ({ workspace }: { workspace: string }) => ({ + enabled: true, + sessions: [ + { id: 's9', updated_at: '2026-09-14T00:00:00Z', epoch: workspace === 'ws' ? 0 : 1 } + ] + })) + pullMock.mockImplementation(async ({ workspace }: { workspace: string }) => ({ + enabled: true, + sessions: [{ ...backup, head: { ...backup.head, workspace_id: workspace, moves: 1 } }], + deferred: [] + })) + usersWorkspaceStore.set({ + email: EMAIL, + workspaces: [ + { id: 'ws', name: 'ws', username: 'u' }, + { id: 'ws2', name: 'ws2', username: 'u', parent_workspace_id: 'ws' } + ] + } as never) + restoreSessionBackups('ws') + await __settleForTesting() + // The stale copy in the old workspace is left alone, and does not take the id first. + expect(pullMock.mock.calls.map((c) => c[0].workspace)).toEqual(['ws2']) + await vi.waitFor(() => expect(sessionState.sessions.map((s) => s.id)).toEqual(['s9'])) + expect(sessionState.sessions[0].workspace_id).toBe('ws2') + }) + + it('leaves a session whose later copy showed up elsewhere after the listings, and tries again', async () => { + // The move lands in the other workspace between the family's listings and the pull: + // the listing taken again before the record lands shows it, and the copy about to be + // imported is the stale one. + let ws2Listings = 0 + listMock.mockImplementation(async ({ workspace }: { workspace: string }) => { + if (workspace === 'ws') { + return { + enabled: true, + sessions: [{ id: 's9', updated_at: '2026-09-14T00:00:00Z', epoch: 0 }] + } + } + ws2Listings += 1 + return { + enabled: true, + sessions: + ws2Listings === 1 ? [] : [{ id: 's9', updated_at: '2026-09-14T00:00:00Z', epoch: 1 }] + } + }) + pullMock.mockImplementation(async ({ workspace }: { workspace: string }) => ({ + enabled: true, + sessions: [{ ...backup, head: { ...backup.head, workspace_id: workspace, moves: 1 } }], + deferred: [] + })) + usersWorkspaceStore.set({ + email: EMAIL, + workspaces: [ + { id: 'ws', name: 'ws', username: 'u' }, + { id: 'ws2', name: 'ws2', username: 'u', parent_workspace_id: 'ws' } + ] + } as never) + restoreSessionBackups('ws') + await __settleForTesting() + expect(pullMock.mock.calls.map((c) => c[0].workspace)).toEqual(['ws']) + expect(sessionState.sessions).toEqual([]) + restoreSessionBackups('ws') + await __settleForTesting() + await vi.waitFor(() => expect(sessionState.sessions.map((s) => s.id)).toEqual(['s9'])) + expect(sessionState.sessions[0].workspace_id).toBe('ws2') + }) + + it('checks a family member whose backups were off when the restore started', async () => { + // The other workspace comes on (and gets the moved session) between the family's + // listings and the pull: the listing taken again before the records land covers it. + let ws2Listings = 0 + listMock.mockImplementation(async ({ workspace }: { workspace: string }) => { + if (workspace === 'ws') { + return { + enabled: true, + sessions: [{ id: 's9', updated_at: '2026-09-14T00:00:00Z', epoch: 0 }] + } + } + ws2Listings += 1 + return ws2Listings === 1 + ? { enabled: false, sessions: [] } + : { + enabled: true, + sessions: [{ id: 's9', updated_at: '2026-09-14T00:00:00Z', epoch: 1 }] + } + }) + pullMock.mockImplementation(async ({ workspace }: { workspace: string }) => ({ + enabled: true, + sessions: [{ ...backup, head: { ...backup.head, workspace_id: workspace, moves: 1 } }], + deferred: [] + })) + usersWorkspaceStore.set({ + email: EMAIL, + workspaces: [ + { id: 'ws', name: 'ws', username: 'u' }, + { id: 'ws2', name: 'ws2', username: 'u', parent_workspace_id: 'ws' } + ] + } as never) + restoreSessionBackups('ws') + await __settleForTesting() + expect(sessionState.sessions).toEqual([]) + restoreSessionBackups('ws') + await __settleForTesting() + await vi.waitFor(() => expect(sessionState.sessions.map((s) => s.id)).toEqual(['s9'])) + expect(sessionState.sessions[0].workspace_id).toBe('ws2') + }) + + it('marks nothing at load for a session a restore brought back', async () => { + listMock.mockResolvedValue({ + enabled: true, + sessions: [{ id: 's9', updated_at: '2026-09-14T00:00:00Z', epoch: 0 }] + }) + pullMock.mockResolvedValue({ enabled: true, sessions: [backup], deferred: [] }) + usersWorkspaceStore.set({ email: EMAIL, workspaces: [] } as never) + restoreSessionBackups('ws') + await __settleForTesting() + await vi.waitFor(() => expect(sessionState.sessions.map((s) => s.id)).toEqual(['s9'])) + // The next load's backfill leaves the clean row alone: no mark, no read, no push. + __resetMirrorForTesting() + await __flushForTesting() + expect(pushMock).not.toHaveBeenCalled() + expect(pendingKeys()).toEqual([]) + }) + + it('lists a family of one once, whatever it restores', async () => { + listMock.mockResolvedValue({ + enabled: true, + sessions: [ + { id: 's9', updated_at: '2026-09-14T00:00:00Z', epoch: 0 }, + { id: 's8', updated_at: '2026-09-13T00:00:00Z', epoch: 0 } + ] + }) + pullMock.mockImplementation(async ({ requestBody }: { requestBody: { ids: string[] } }) => ({ + enabled: true, + sessions: requestBody.ids.map((id) => ({ + ...backup, + id, + head: { ...backup.head, id }, + chats: backup.chats.map((c) => ({ + ...c, + id: `${c.id}-${id}`, + record: { ...c.record, id: `${c.id}-${id}`, sessionId: id } + })) + })), + deferred: [] + })) + usersWorkspaceStore.set({ email: EMAIL, workspaces: [] } as never) + restoreSessionBackups('ws') + await __settleForTesting() + await vi.waitFor(() => + expect(sessionState.sessions.map((s) => s.id).sort()).toEqual(['s8', 's9']) + ) + expect(listMock).toHaveBeenCalledTimes(1) + }) + + it('restores nothing of a family one of whose workspaces could not be listed, and tries again', async () => { + listMock + .mockImplementationOnce(async () => ({ + enabled: true, + sessions: [{ id: 's9', updated_at: '2026-09-14T00:00:00Z', epoch: 0 }] + })) + .mockRejectedValueOnce(new Error('offline')) + .mockImplementation(async ({ workspace }: { workspace: string }) => ({ + enabled: true, + sessions: [ + { id: 's9', updated_at: '2026-09-14T00:00:00Z', epoch: workspace === 'ws' ? 0 : 1 } + ] + })) + pullMock.mockImplementation(async ({ workspace }: { workspace: string }) => ({ + enabled: true, + sessions: [{ ...backup, head: { ...backup.head, workspace_id: workspace, moves: 1 } }], + deferred: [] + })) + usersWorkspaceStore.set({ + email: EMAIL, + workspaces: [ + { id: 'ws', name: 'ws', username: 'u' }, + { id: 'ws2', name: 'ws2', username: 'u', parent_workspace_id: 'ws' } + ] + } as never) + restoreSessionBackups('ws') + await __settleForTesting() + // The copy that listed could be the stale one: nothing is imported this time. + expect(pullMock).not.toHaveBeenCalled() + expect(sessionState.sessions).toEqual([]) + restoreSessionBackups('ws') + await __settleForTesting() + expect(pullMock.mock.calls.map((c) => c[0].workspace)).toEqual(['ws2']) + await vi.waitFor(() => expect(sessionState.sessions.map((s) => s.id)).toEqual(['s9'])) + }) + + it('starts a session over when its backup moved between two pages', async () => { + listMock.mockResolvedValue({ + enabled: true, + sessions: [{ id: 's9', updated_at: '2026-09-14T00:00:00Z' }] + }) + const cursor = { id: 's9', images: false, after: 'sessions/s9/chats/c9.json' } + const c9b = { ...backup.chats[0], id: 'c9b', record: { ...backup.chats[0].record, id: 'c9b' } } + const c9a = { ...backup.chats[0], id: 'c9a', record: { ...backup.chats[0].record, id: 'c9a' } } + pullMock + // The first attempt: a chat sorting before the cursor lands between the pages. + .mockResolvedValueOnce({ + enabled: true, + sessions: [{ ...backup, next: cursor, listing: 'L1' }], + deferred: [] + }) + .mockResolvedValueOnce({ + enabled: true, + sessions: [{ ...backup, chats: [c9b], listing: 'L2' }], + deferred: [] + }) + // The second attempt sees the whole of it. + .mockResolvedValueOnce({ + enabled: true, + sessions: [{ ...backup, chats: [c9a, backup.chats[0]], next: cursor, listing: 'L2' }], + deferred: [] + }) + .mockResolvedValueOnce({ + enabled: true, + sessions: [{ ...backup, chats: [c9b], listing: 'L2' }], + deferred: [] + }) + usersWorkspaceStore.set({ email: EMAIL, workspaces: [] } as never) + restoreSessionBackups('ws') + await __settleForTesting() + expect(pullMock).toHaveBeenCalledTimes(4) + expect(pullMock.mock.calls[2][0].requestBody).toEqual({ ids: ['s9'], resume: undefined }) + await vi.waitFor(() => expect(sessionState.sessions.map((s) => s.id)).toEqual(['s9'])) + expect((await readStoredChat('c9a', EMAIL))?.id).toBe('c9a') + expect((await readStoredChat('c9b', EMAIL))?.id).toBe('c9b') + }) + + it('restores nothing without Web Locks, and still backs up', async () => { + setWebLocks(undefined) + listMock.mockResolvedValue({ + enabled: true, + sessions: [{ id: 's9', updated_at: '2026-09-14T00:00:00Z' }] + }) + usersWorkspaceStore.set({ email: EMAIL, workspaces: [] } as never) + restoreSessionBackups('ws') + await __settleForTesting() + expect(listMock).not.toHaveBeenCalled() + expect(sessionState.sessions).toEqual([]) + const s: Session = { id: 'sl', name: 'session-1', createdAt: 1, workspace_id: 'ws' } + sessionState.sessions = [s] + await putSession(s) + pushMock.mockResolvedValueOnce({ enabled: true, results: [{ id: 'sl' }] }) + await __flushForTesting() + expect(pushMock).toHaveBeenCalledTimes(1) + }) + + it('leaves a session for the next restore when what an earlier one staged cannot be pruned', async () => { + listMock.mockResolvedValue({ + enabled: true, + sessions: [{ id: 's9', updated_at: '2026-09-14T00:00:00Z' }] + }) + // An earlier restore was cut short after staging a chat the backup has since dropped. + const { importStoredChats } = await import('../copilot/chat/HistoryManager.svelte') + await importStoredChats([{ ...backup.chats[0].record, id: 'cx' } as never], [], EMAIL, true) + const staging = { + id: 's9', + ws: 'ws', + head: '', + chats: {}, + images: {}, + staging: { chats: ['cx'], images: [], items: [], versions: [] } + } + await __writeSyncForTesting([staging], EMAIL) + pullMock.mockResolvedValue({ enabled: true, sessions: [backup], deferred: [] }) + chatImport.pruneFails = true + usersWorkspaceStore.set({ email: EMAIL, workspaces: [] } as never) + restoreSessionBackups('ws') + await __settleForTesting() + // No record, so no flush can push the stale chat back, and the staging row stays, + // now naming what this page wrote too. + expect(sessionState.sessions).toEqual([]) + expect((await readStoredChat('cx', EMAIL))?.id).toBe('cx') + expect( + (await __syncRowsForTesting(EMAIL)).find((r) => r.id === 's9')?.staging?.chats?.sort() + ).toEqual(['c9', 'cx']) + + // The restore after prunes and brings the session back. + chatImport.pruneFails = false + __resetMirrorForTesting() + restoreSessionBackups('ws') + await __settleForTesting() + await vi.waitFor(() => expect(sessionState.sessions.map((s) => s.id)).toEqual(['s9'])) + expect(await readStoredChat('cx', EMAIL)).toBeUndefined() + expect((await __syncRowsForTesting(EMAIL)).find((r) => r.id === 's9')?.staging).toBeUndefined() + }) + + it('never writes an older record over a newer one, and prunes only what it is told', async () => { + const { importStoredChats, pruneSessionChats } = await import( + '../copilot/chat/HistoryManager.svelte' + ) + const chat = (lastModified: number, title: string) => + ({ ...backup.chats[0].record, lastModified, title }) as never + await importStoredChats([chat(20, 'newer')], [], EMAIL, true) + await importStoredChats([chat(10, 'older')], [], EMAIL, true) + expect((await readStoredChat('c9', EMAIL))?.title).toBe('newer') + await importStoredChats( + [{ ...backup.chats[0].record, id: 'c9b', lastModified: 30 } as never], + [], + EMAIL, + true + ) + // A prune names what goes; nothing else of the session is touched. + await pruneSessionChats('s9', new Set(['c9']), new Set(), EMAIL) + expect(await readStoredChat('c9', EMAIL)).toBeUndefined() + expect((await readStoredChat('c9b', EMAIL))?.id).toBe('c9b') + }) + + it('keeps an image whose chat came on an earlier page, and restages after a page failed', async () => { + listMock.mockResolvedValue({ + enabled: true, + sessions: [{ id: 's9', updated_at: '2026-09-14T00:00:00Z' }] + }) + const cursor = { id: 's9', images: true, after: '' } + const imagePage = { + ...backup, + chats: [], + images: [{ chat_id: 'c9', id: 'i9', data_url: IMAGE }] + } + // The first restore is cut short after staging the chats and an artifact with two + // versions. + const gone = { ...backup.chats[0], id: 'cx', record: { ...backup.chats[0].record, id: 'cx' } } + const item = { id: 'a1', sessionId: 's9', kind: 'markdown', name: 'a', content: 'x' } + const version = (n: number) => ({ + key: `a1:${n}`, + artifactId: 'a1', + version: n, + name: 'a', + content: 'x', + savedAt: n + }) + pullMock + .mockResolvedValueOnce({ + enabled: true, + sessions: [ + { + ...backup, + chats: [...backup.chats, gone], + artifacts: { items: [item], versions: [version(1), version(2)] }, + next: cursor + } + ], + deferred: [] + }) + .mockRejectedValueOnce(new Error('offline')) + usersWorkspaceStore.set({ email: EMAIL, workspaces: [] } as never) + restoreSessionBackups('ws') + await __settleForTesting() + expect(pullMock).toHaveBeenCalledTimes(2) + expect(sessionState.sessions.map((s) => s.id)).toEqual([]) + expect((await readStoredChat('c9', EMAIL))?.title).toBe('t') + expect((await readStoredChat('cx', EMAIL))?.id).toBe('cx') + + // The backup moved on meanwhile: the retry takes the newer chat over the staged one + // and drops the chat the backup no longer has. + __resetMirrorForTesting() + const newer = { + ...backup, + chats: [{ ...backup.chats[0], record: { ...backup.chats[0].record, title: 'newer' } }], + artifacts: { items: [item], versions: [version(1)] } + } + pullMock + .mockResolvedValueOnce({ + enabled: true, + sessions: [{ ...newer, next: cursor }], + deferred: [] + }) + .mockResolvedValueOnce({ enabled: true, sessions: [imagePage], deferred: [] }) + restoreSessionBackups('ws') + await __settleForTesting() + await vi.waitFor(() => expect(sessionState.sessions.map((s) => s.id)).toEqual(['s9'])) + expect((await readStoredChat('c9', EMAIL))?.title).toBe('newer') + expect(await readStoredChat('cx', EMAIL)).toBeUndefined() + const { readImageDataUrl } = await import('../copilot/chat/HistoryManager.svelte') + expect(await readImageDataUrl('i9', EMAIL)).toBe(IMAGE) + // The version the backup no longer has went with the chat it no longer has. + const { readSessionArtifacts } = await import('../copilot/chat/artifacts/artifactsDB') + const artifacts = await readSessionArtifacts('s9', EMAIL) + expect(artifacts?.items.map((i) => i.id)).toEqual(['a1']) + expect(artifacts?.versions.map((v) => v.key)).toEqual(['a1:1']) + }) +}) diff --git a/frontend/src/lib/components/sessions/sessionMirrorPlan.test.ts b/frontend/src/lib/components/sessions/sessionMirrorPlan.test.ts new file mode 100644 index 0000000000..aa3c8bf735 --- /dev/null +++ b/frontend/src/lib/components/sessions/sessionMirrorPlan.test.ts @@ -0,0 +1,204 @@ +import { describe, expect, it } from 'vitest' +import { + artifactsFingerprint, + headSig, + jsonBytes, + planSessionPush, + splitEntry, + type ChatSnapshot, + type MirrorSyncState +} from './sessionMirrorPlan' +import type { Session } from './sessionState.svelte' + +function session(over: Partial = {}): Session { + return { id: 's1', name: 'session-1', createdAt: 1, workspace_id: 'ws', chatId: 'c1', ...over } +} + +function chat(id: string, lastModified: number, imageIds: string[] = []): ChatSnapshot { + return { id, lastModified, record: { id, lastModified }, imageIds } +} + +const noArtifacts = { items: [], versions: [] } + +function synced(over: Partial = {}): MirrorSyncState { + return { + id: 's1', + ws: 'ws', + head: headSig(session()), + chats: { c1: 10 }, + images: { i1: 'c1' }, + artifacts: artifactsFingerprint(noArtifacts), + ...over + } +} + +describe('planSessionPush', () => { + it('pushes everything for a session never backed up, and nothing for an unsent draft', () => { + const plan = planSessionPush({ + session: session(), + chats: [chat('c1', 10, ['i1'])], + artifacts: noArtifacts + }) + expect(plan?.entry?.head?.id).toBe('s1') + expect(plan?.entry?.chats?.map((c) => c.id)).toEqual(['c1']) + expect(plan?.images).toEqual([{ chat_id: 'c1', id: 'i1' }]) + // Nothing to store yet, so no artifacts object either. + expect(plan?.entry?.artifacts).toBeUndefined() + expect(plan?.next).toEqual(synced()) + + expect( + planSessionPush({ + session: session({ workspace_id: undefined, pending_workspace_id: 'ws' }), + chats: [], + artifacts: noArtifacts + }) + ).toBeUndefined() + }) + + it('sends nothing when only the fields reading a session bumps changed', () => { + const plan = planSessionPush({ + session: session({ lastSeenCount: 7, lastActivityAt: 99, name: 'session-9' }), + chats: [chat('c1', 10, ['i1'])], + artifacts: noArtifacts, + sync: synced() + }) + expect(plan?.entry).toBeUndefined() + expect(plan?.images).toEqual([]) + }) + + it('carries only the chat whose lastModified moved, plus its new images', () => { + const plan = planSessionPush({ + session: session(), + chats: [chat('c1', 10, ['i1']), chat('c2', 20, ['i2'])], + artifacts: noArtifacts, + sync: synced() + }) + expect(plan?.entry?.head).toBeUndefined() + expect(plan?.entry?.chats?.map((c) => c.id)).toEqual(['c2']) + expect(plan?.images).toEqual([{ chat_id: 'c2', id: 'i2' }]) + expect(plan?.next.chats).toEqual({ c1: 10, c2: 20 }) + expect(plan?.next.images).toEqual({ i1: 'c1', i2: 'c2' }) + }) + + it('deletes the copy of a chat that grew too large to back up, instead of keeping a stale one', () => { + const plan = planSessionPush({ + session: session(), + chats: [{ id: 'c1', lastModified: 11, imageIds: ['i1'], omitted: true }], + artifacts: noArtifacts, + sync: synced() + }) + expect(plan?.entry?.delete_chats).toEqual(['c1']) + expect(plan?.entry?.chats).toBeUndefined() + expect(plan?.images).toEqual([]) + expect(plan?.next.chats).toEqual({}) + expect(plan?.next.images).toEqual({}) + }) + + it('carries deletes past the per-entry cap over to the next push', () => { + const prevChats = Object.fromEntries(Array.from({ length: 1005 }, (_, i) => [`c${i}`, 10])) + const plan = planSessionPush({ + session: session(), + chats: [], + artifacts: noArtifacts, + sync: synced({ chats: prevChats, images: {} }) + }) + expect(plan?.entry?.delete_chats).toHaveLength(1000) + // Still listed as pushed, so the next plan finds them gone again, and the session + // stays marked for that plan. + expect(Object.keys(plan?.next.chats ?? {})).toHaveLength(5) + expect(plan?.carried).toBe(true) + }) + + it('deletes a chat that is gone and an image its chat evicted', () => { + const plan = planSessionPush({ + session: session(), + chats: [chat('c1', 11, [])], + artifacts: noArtifacts, + sync: synced({ chats: { c1: 10, c2: 20 }, images: { i1: 'c1', i2: 'c2' } }) + }) + expect(plan?.entry?.delete_chats).toEqual(['c2']) + // i2 goes with c2 server-side; only c1's evicted image is deleted on its own. + expect(plan?.entry?.delete_images).toEqual([{ chat_id: 'c1', id: 'i1' }]) + }) + + it('moves a session as a full push to the new workspace and a removal from the old', () => { + const plan = planSessionPush({ + session: session({ workspace_id: 'ws2' }), + chats: [chat('c1', 10, ['i1'])], + artifacts: noArtifacts, + sync: synced() + }) + expect(plan?.workspaceId).toBe('ws2') + expect(plan?.removeFrom).toBe('ws') + expect(plan?.entry?.head?.workspace_id).toBe('ws2') + expect(plan?.entry?.chats?.map((c) => c.id)).toEqual(['c1']) + expect(plan?.images).toEqual([{ chat_id: 'c1', id: 'i1' }]) + expect(plan?.next.ws).toBe('ws2') + }) + + it('pushes artifacts when their fingerprint changes, including emptying them', () => { + const items = [ + { + id: 'a1', + sessionId: 's1', + kind: 'md' as const, + name: 'notes', + content: 'x', + createdAt: 1, + updatedAt: 2, + version: 1 + } + ] + const withArtifact = planSessionPush({ + session: session(), + chats: [chat('c1', 10, ['i1'])], + artifacts: { items, versions: [] }, + sync: synced() + }) + expect(withArtifact?.entry?.artifacts).toEqual({ items, versions: [] }) + + const emptied = planSessionPush({ + session: session(), + chats: [chat('c1', 10, ['i1'])], + artifacts: noArtifacts, + sync: synced({ artifacts: artifactsFingerprint({ items, versions: [] }) }) + }) + expect(emptied?.entry?.artifacts).toEqual(noArtifacts) + }) +}) + +describe('jsonBytes', () => { + it('counts the bytes the request carries, not UTF-16 code units', () => { + expect(jsonBytes('ab')).toBe(4) + expect(jsonBytes('日本')).toBe(8) + expect(jsonBytes('😀')).toBe(6) + }) +}) + +describe('splitEntry', () => { + it('splits an oversized entry into chat-only parts, the head riding on the last', () => { + const big = (id: string) => ({ id, record: { id, text: 'x'.repeat(150) } }) + const entry = { + id: 's', + head: { id: 's' }, + chats: [big('c1'), big('c2'), big('c3')], + delete_chats: ['old'] + } + const parts = splitEntry(entry, 200) + expect(parts.map((p) => p.chats?.map((c) => c.id))).toEqual([['c1'], ['c2'], ['c3']]) + expect( + parts.slice(0, -1).every((p) => p.head === undefined && p.delete_chats === undefined) + ).toBe(true) + expect(parts.at(-1)?.head).toEqual({ id: 's' }) + expect(parts.at(-1)?.delete_chats).toEqual(['old']) + // Within the target, or a single chat: nothing to split. + expect(splitEntry(entry, 10_000)).toEqual([entry]) + expect(splitEntry({ id: 's', chats: [big('c1')] }, 10)).toHaveLength(1) + // The server's per-entry chat cap splits too, however small the chats. + const many = { + id: 's', + chats: Array.from({ length: 250 }, (_, i) => ({ id: `c${i}`, record: {} })) + } + expect(splitEntry(many, 1_000_000).map((p) => p.chats?.length)).toEqual([100, 100, 50]) + }) +}) diff --git a/frontend/src/lib/components/sessions/sessionMirrorPlan.ts b/frontend/src/lib/components/sessions/sessionMirrorPlan.ts new file mode 100644 index 0000000000..c61bcf2763 --- /dev/null +++ b/frontend/src/lib/components/sessions/sessionMirrorPlan.ts @@ -0,0 +1,309 @@ +// The pure half of the session backup: given what the local stores hold for one session +// and what was last pushed, decide what the next push carries. Every piece is compared +// against its own marker (a signature for the record, `lastModified` for a chat, the id +// for a write-once image) so a session that only changed locally in ways the backup +// does not keep sends nothing. +import type { AISessionBackupPush } from '$lib/gen' +import { orderedJsonStringify } from '$lib/utils' +import type { Session } from './sessionState.svelte' +import type { ArtifactVersion, PersistedArtifact } from '../copilot/chat/artifacts/artifactsDB' + +/** What the backup remembers of a session after a successful push. */ +export interface MirrorSyncState { + id: string + /** The workspace whose storage holds the backup. */ + ws: string + /** `headSig` of the record pushed. */ + head: string + /** `lastModified` of each chat pushed, by chat id. */ + chats: Record + /** The chat each pushed image belongs to, by image id. */ + images: Record + artifacts?: string + /** The workspace's storage went away after this push: what it holds is unknown, so the + * next push carries everything again. Kept rather than deleted, so a removal still + * knows a backup existed. */ + stale?: boolean + /** The dirty mark's counter this push covered. A mark is pending while its counter is + * above this; retiring it here rather than deleting the mark means a tab bumping the + * counter while another flushes can never have its bump erased. */ + flushedV?: number + /** The user deleted the session and its removal mark could not be written to + * localStorage (full): the row itself carries the removal, until it lands. */ + removed?: boolean + /** The storage the push landed in, as the server names it, and the backup generation + * (bumped by a workspace key rotation) it landed under. A row recorded against another + * storage or generation describes objects the server no longer looks at. */ + storageId?: string + generation?: number + /** Other storages this workspace was on that still hold a copy of the backup (a switch + * leaves the old copy where it was): a removal is done only once each has answered it, + * or a switch back would bring a deleted session back. */ + alsoIn?: string[] + /** Bumps of the dirty mark that localStorage refused, recorded here instead: the mark's + * counter plus this is what a push retires, and every row write keeps it. */ + extraV?: number + /** A restore in progress (or cut short): the pieces it wrote for a session that has no + * record yet, so a later restore deletes the ones the backup no longer has. */ + staging?: { chats: string[]; images: string[]; items: string[]; versions: string[] } +} + +/** + * The part of a session record the backup keeps. Left out on purpose: `name` (a + * per-browser counter), the unsent-draft fields (`pending_*`, `draftPrompt`, + * `autoSendDraftAt`), `workspace_root_id` (derived on import), `transient`, and the two + * fields reading a session bumps (`lastSeenCount`, `lastActivityAt`) — so opening a + * session and reading its new messages never costs a push. + */ +export type SessionHead = Pick< + Session, + | 'id' + | 'workspace_id' + | 'chatId' + | 'summary' + | 'summarySource' + | 'createdAt' + | 'archived' + | 'archivedByWorkspace' + | 'moves' + | 'previewTabs' + | 'activePreviewTabId' + | 'previewCollapsed' + | 'previewSize' +> + +export function sessionHead(s: Session): SessionHead { + const head: SessionHead = { id: s.id, createdAt: s.createdAt } + if (s.workspace_id !== undefined) head.workspace_id = s.workspace_id + if (s.chatId !== undefined) head.chatId = s.chatId + if (s.summary !== undefined) head.summary = s.summary + if (s.summarySource !== undefined) head.summarySource = s.summarySource + if (s.archived !== undefined) head.archived = s.archived + if (s.archivedByWorkspace !== undefined) head.archivedByWorkspace = s.archivedByWorkspace + if (s.moves !== undefined) head.moves = s.moves + if (s.previewTabs !== undefined) head.previewTabs = s.previewTabs + if (s.activePreviewTabId !== undefined) head.activePreviewTabId = s.activePreviewTabId + if (s.previewCollapsed !== undefined) head.previewCollapsed = s.previewCollapsed + if (s.previewSize !== undefined) head.previewSize = s.previewSize + return head +} + +export function headSig(s: Session): string { + return orderedJsonStringify(sessionHead(s)) +} + +export interface ArtifactsSnapshot { + items: PersistedArtifact[] + versions: ArtifactVersion[] +} + +/** Cheap to compute from the rows alone: every edit bumps `updatedAt`, every snapshot has + * its own key, and approving a plan changes `approvedVersion`. */ +export function artifactsFingerprint(a: ArtifactsSnapshot): string { + const items = a.items + .map((i) => `${i.id}:${i.updatedAt}:${i.version ?? 1}:${i.approvedVersion ?? ''}`) + .sort() + const versions = a.versions.map((v) => v.key).sort() + return JSON.stringify([items, versions]) +} + +export interface ChatSnapshot { + id: string + lastModified: number + /** The stored record; absent for a chat that did not change since the last push, whose + * bytes the caller did not read. */ + record?: unknown + imageIds: string[] + /** Too large to back up: planned as if it did not exist, so a copy pushed while it was + * smaller is deleted rather than restored one day as the current transcript. */ + omitted?: boolean +} + +export interface PlanInput { + session: Session + /** Every chat the session owns right now. */ + chats: ChatSnapshot[] + artifacts: ArtifactsSnapshot + sync?: MirrorSyncState +} + +export interface PlannedPush { + workspaceId: string + /** Absent when nothing changed that the backup keeps. */ + entry?: AISessionBackupPush + /** Images the entry needs uploaded, whose bytes the caller loads. */ + images: { chat_id: string; id: string }[] + /** The workspace the session was backed up in before it moved. */ + removeFrom?: string + next: MirrorSyncState + /** Deletes past the per-entry cap were left in `next` for the following push, so the + * session must stay marked once this one lands. */ + carried: boolean + /** Nothing of the session is taken to be in the storage: every piece goes, and the + * first part opens the push whole (see `whole` on the entry). */ + whole: boolean +} + +/** `undefined` for a session with nowhere to go: an unsent draft has no workspace yet. */ +export function planSessionPush(input: PlanInput): PlannedPush | undefined { + const { session, chats, artifacts } = input + const workspaceId = session.workspace_id + if (!workspaceId) return undefined + // A move is a full push into the new workspace's storage; the copy in the old one goes. + const prev = input.sync?.ws === workspaceId ? input.sync : undefined + const removeFrom = input.sync && input.sync.ws !== workspaceId ? input.sync.ws : undefined + + const entry: AISessionBackupPush = { id: session.id } + let changed = false + let carried = false + const sig = headSig(session) + if (prev?.head !== sig) { + entry.head = sessionHead(session) + changed = true + } + + const next: MirrorSyncState = { + id: session.id, + ws: workspaceId, + head: sig, + chats: {}, + images: {} + } + const images: { chat_id: string; id: string }[] = [] + const pushedChats: { id: string; record: Record }[] = [] + for (const chat of chats) { + if (chat.omitted) continue + next.chats[chat.id] = chat.lastModified + if (prev?.chats[chat.id] !== chat.lastModified && chat.record !== undefined) { + pushedChats.push({ id: chat.id, record: chat.record as Record }) + } + for (const id of chat.imageIds) { + next.images[id] = chat.id + if (prev?.images[id] === undefined) images.push({ chat_id: chat.id, id }) + } + } + if (pushedChats.length > 0) { + entry.chats = pushedChats + changed = true + } + if (prev) { + const gone = Object.keys(prev.chats).filter((id) => next.chats[id] === undefined) + // An image evicted by the per-chat cap, from a chat that is still there (a deleted + // chat takes its images with it server-side). + const evicted = Object.entries(prev.images).filter( + ([id, chatId]) => next.images[id] === undefined && next.chats[chatId] !== undefined + ) + // Past the server's cap per entry, the rest stays in `next` as if still pushed, so + // the following push finds it gone again. + if (gone.length > 0) { + entry.delete_chats = gone.slice(0, MAX_DELETES_PER_ENTRY) + for (const id of gone.slice(MAX_DELETES_PER_ENTRY)) { + next.chats[id] = prev.chats[id] + carried = true + } + changed = true + } + if (evicted.length > 0) { + entry.delete_images = evicted + .slice(0, MAX_DELETES_PER_ENTRY) + .map(([id, chatId]) => ({ chat_id: chatId, id })) + for (const [id, chatId] of evicted.slice(MAX_DELETES_PER_ENTRY)) { + next.images[id] = chatId + carried = true + } + changed = true + } + } + + const fingerprint = artifactsFingerprint(artifacts) + next.artifacts = fingerprint + if (prev?.artifacts !== fingerprint && (artifacts.items.length > 0 || prev?.artifacts)) { + entry.artifacts = { items: artifacts.items, versions: artifacts.versions } + changed = true + } + + return { + workspaceId, + entry: changed ? entry : undefined, + images, + removeFrom, + next, + carried, + whole: prev === undefined + } +} + +/** Bytes a JSON body would carry for this value, as sent: UTF-8, not UTF-16 code units, + * which would under-count a transcript in a non-Latin script by up to three times. Counted + * rather than encoded: the values measured are the multi-megabyte ones. */ +export function jsonBytes(value: unknown): number { + const text = JSON.stringify(value) + let bytes = 0 + for (let i = 0; i < text.length; i++) { + const c = text.charCodeAt(i) + if (c < 0x80) bytes += 1 + else if (c < 0x800) bytes += 2 + else if (c >= 0xd800 && c <= 0xdbff) { + // A surrogate pair is one four-byte code point. + bytes += 4 + i++ + } else bytes += 3 + } + return bytes +} + +/** Object-store calls the server makes for an entry, the unit its per-request cap counts. */ +export function operationsOf(entry: AISessionBackupPush): number { + return ( + (entry.chats?.length ?? 0) + + (entry.images?.length ?? 0) + + (entry.delete_chats?.length ?? 0) + + (entry.delete_images?.length ?? 0) + ) +} + +export interface PushBody { + owner: string + sessions: AISessionBackupPush[] + removed?: string[] +} + +/** The server's caps on chats and on each delete list per entry. */ +export const MAX_CHATS_PER_ENTRY = 100 +export const MAX_DELETES_PER_ENTRY = 1000 + +/** + * Break an entry that outgrows the target, or the server's per-entry chat cap, into + * chat-only entries, each written on its own, with everything else riding on the last one: + * the entries go out in order and the server lists the session by the last, so the marker + * never lists a chat that has not landed. (A push of the session whole moves the head to + * whichever part goes first; see the mirror.) + */ +export function splitEntry(entry: AISessionBackupPush, targetBytes: number): AISessionBackupPush[] { + if ( + !entry.chats || + entry.chats.length <= 1 || + (entry.chats.length <= MAX_CHATS_PER_ENTRY && jsonBytes(entry) <= targetBytes) + ) { + return [entry] + } + const { chats, ...rest } = entry + const parts: AISessionBackupPush[] = [] + let current: typeof chats = [] + let size = 0 + for (const chat of chats) { + const bytes = jsonBytes(chat) + if ( + current.length > 0 && + (current.length >= MAX_CHATS_PER_ENTRY || size + bytes > targetBytes) + ) { + parts.push({ id: entry.id, chats: current }) + current = [] + size = 0 + } + current.push(chat) + size += bytes + } + parts.push({ ...rest, chats: current }) + return parts +} diff --git a/frontend/src/lib/components/sessions/sessionMirrorSignal.ts b/frontend/src/lib/components/sessions/sessionMirrorSignal.ts new file mode 100644 index 0000000000..eb936d2e22 --- /dev/null +++ b/frontend/src/lib/components/sessions/sessionMirrorSignal.ts @@ -0,0 +1,42 @@ +// The one-way channel from the IndexedDB write funnels (session records, chat history, +// artifacts) to the session backup. Import-free on purpose: the stores it is called from +// must not depend on the backup module, which depends on all of them. + +// `email` names the user whose store the write landed in (from the store's scoped name): +// the current user may have changed while the write was pending, and the mark belongs to +// the store's user, not to whoever is logged in when it completes. +export type MirrorSignal = + | { kind: 'dirty'; sessionId: string; chatId?: string; email?: string } + | { kind: 'removed'; sessionId: string; workspaceId?: string; email?: string } + +let handler: ((signal: MirrorSignal) => void) | undefined +// Signals raised before the backup module registered, replayed to it on registration. +let buffered: MirrorSignal[] = [] + +function emit(signal: MirrorSignal): void { + if (handler) handler(signal) + else buffered.push(signal) +} + +/** A durable local write landed for this session (and, when known, this chat) in the + * store of `email`. */ +export function markSessionDirty(sessionId: string, chatId?: string, email?: string): void { + emit({ kind: 'dirty', sessionId, chatId, email }) +} + +/** The user deleted this session; its backup goes with it. */ +export function markSessionRemoved(sessionId: string, workspaceId?: string, email?: string): void { + emit({ kind: 'removed', sessionId, workspaceId, email }) +} + +export function onMirrorSignal(fn: (signal: MirrorSignal) => void): void { + handler = fn + const replay = buffered + buffered = [] + for (const signal of replay) fn(signal) +} + +export function __resetMirrorSignalForTesting(): void { + handler = undefined + buffered = [] +} diff --git a/frontend/src/lib/components/sessions/sessionState.svelte.ts b/frontend/src/lib/components/sessions/sessionState.svelte.ts index d19faeac92..2670b3fa4d 100644 --- a/frontend/src/lib/components/sessions/sessionState.svelte.ts +++ b/frontend/src/lib/components/sessions/sessionState.svelte.ts @@ -24,8 +24,10 @@ import { workspaceRootId } from './sessionScope.svelte' import { clearSessionRecovered } from './sessionRecoveryNotice.svelte' import { type DBSchema, type IDBPDatabase } from 'idb' import { userScopedDb } from '$lib/userScopedDb' +import { emailOfScopedKey, scopedKeyFor } from '$lib/userScopedStorage' import { deleteItemsForSession } from '../copilot/chat/files/attachedFilesDB' import { deleteArtifactsForSession } from '../copilot/chat/artifacts/artifactsDB' +import { markSessionDirty, markSessionRemoved } from './sessionMirrorSignal' // Switch the global workspace iff the target differs from the active one // and is non-empty. Centralises the "session needs its workspace in focus" @@ -102,6 +104,10 @@ export type Session = { // archived (not by the user). Lets reconciliation auto-unarchive the session // when the workspace is unarchived, while leaving user-archived sessions be. archivedByWorkspace?: boolean + // How many times the session moved to another workspace. The backup keeps it + // with the session's marker, so a restore that finds a copy in two workspaces + // (moved, the old copy not yet removed) takes the later one without a clock. + moves?: number // In-memory-only flag: the session exists but hasn't been written to // IndexedDB yet. Set at creation, cleared on the first genuine user touch // (typed prompt, workspace/fork pick, preview tab, rename) which persists @@ -444,6 +450,7 @@ async function deleteSessionRow(db: IDBPDatabase, id: string): Pr async function putSessionRow(db: IDBPDatabase, s: Session): Promise { if (deletedSessionIds.has(s.id)) return await db.put('sessions', s) + markSessionDirty(s.id, undefined, emailOfScopedKey(SESSIONS_DB, db.name)) } // Write-behind a single session record. Transient sessions are in-memory only @@ -775,6 +782,15 @@ export function findEmptyLandingSession(): Session | undefined { ) } +// Session names are a per-browser counter (`session-N`) that the sessions page puts in +// its URL, so a new or restored record takes the number after the highest in use. +function nextSessionNumber(sessions: Session[]): number { + const numbers = sessions + .map((s) => /^session-(\d+)$/.exec(s.name)?.[1]) + .map((n) => (n ? parseInt(n, 10) : 0)) + return (numbers.length ? Math.max(...numbers) : 0) + 1 +} + export function createSession(): Session { // Reuse an existing untouched draft from the active family rather than pile a // blank entry on every `+`, so several pending sessions can still be built up @@ -795,10 +811,7 @@ export function createSession(): Session { return reusable } sessionState.sessions = sessionState.sessions.filter((s) => !isDiscardableDraft(s)) - const existingNumbers = sessionState.sessions - .map((s) => /^session-(\d+)$/.exec(s.name)?.[1]) - .map((n) => (n ? parseInt(n, 10) : 0)) - const next = (existingNumbers.length ? Math.max(...existingNumbers) : 0) + 1 + const next = nextSessionNumber(sessionState.sessions) // Start in the workspace you're in. The one exception: a root you can't // deploy to (locked, no bypass) steers to its dev, since a session there // couldn't edit anything. The picker lets you switch. @@ -1069,6 +1082,7 @@ export async function moveSessionToWorkspace(id: string, newWorkspaceId: string) const s = sessionState.sessions.find((x) => x.id === id) if (!s) return if (s.workspace_id === newWorkspaceId) return + if (s.workspace_id !== undefined) s.moves = (s.moves ?? 0) + 1 s.workspace_id = newWorkspaceId delete s.pending_workspace_id delete s.pending_fork @@ -1144,9 +1158,63 @@ export function deleteSession(id: string) { // GC any linked files and artifacts persisted for this session. void deleteItemsForSession(id) void deleteArtifactsForSession(id) + // Only a delete the user asked for takes the backup with it: the workspace-lifecycle + // removals above keep theirs, so a session dropped by a wrong reconcile can be restored. + markSessionRemoved(id, s.workspace_id) logFeatureUsage('ai_session', 'deleted', { entityId: id, workspace: s.workspace_id }) } +// --- Session backup support (sessionMirror) --- + +export function isSessionTombstoned(id: string): boolean { + return deletedSessionIds.has(id) +} + +// Every stored record of the named user, or undefined when the store is unavailable or +// already serves someone else: the backup captures its user up front and must not follow +// an in-place account switch. +export async function readStoredSessions(email: string): Promise { + if (!BROWSER) return undefined + const db = await sessionsDb.whenReady() + if (!db || db.name !== scopedKeyFor(SESSIONS_DB, email)) return undefined + try { + return await db.getAll('sessions') + } catch (e) { + console.error('Failed to read sessions from IndexedDB', e) + return undefined + } +} + +// Add restored records for sessions this browser does not have, and re-hydrate the list. +// A record that exists, or was deleted here, is left alone: the local copy is the newer +// one. Returns the ids written. +export async function importSessions(records: Session[], email: string): Promise { + if (!BROWSER) return [] + const db = await sessionsDb.whenReady() + if (!db || db.name !== scopedKeyFor(SESSIONS_DB, email)) return [] + const imported: string[] = [] + try { + const tx = db.transaction('sessions', 'readwrite') + const existing = new Set((await tx.store.getAllKeys()).map(String)) + let next = nextSessionNumber([...(await tx.store.getAll()), ...sessionState.sessions]) + for (const r of records) { + if (existing.has(r.id) || deletedSessionIds.has(r.id)) continue + const record: Session = { ...r, name: `session-${next++}` } + delete record.transient + delete record.workspace_root_id + ensureSessionRootId(record) + await tx.store.put(record) + imported.push(record.id) + } + await tx.done + } catch (e) { + console.error('Failed to import sessions', e) + return [] + } + if (imported.length > 0) await hydrateSessions() + return imported +} + export function setSessionChatId(sessionId: string, chatId: string) { const s = sessionState.sessions.find((x) => x.id === sessionId) if (s && s.chatId !== chatId) { @@ -1172,6 +1240,7 @@ async function patchStoredSessionChatId(s: Session, chatId: string): Promise { userStore.set(undefined) await vi.waitFor(() => expect(sessionState.sessions).toEqual([])) }) + + // A restored backup must never replace what this browser has, come back after the + // user deleted it here, or take a name the sessions page already routes by. + it('importSessions adds only unknown, undeleted records under fresh names', async () => { + const user = freshUser() + await login(user) + const local = session({ id: 'local', name: 'session-3', createdAt: 1, summary: 'mine' }) + await putSession(local) + sessionState.sessions.push(local) + deleteSession('local') + await flush() + await putSession(session({ id: 'kept', name: 'session-5', createdAt: 2, summary: 'kept' })) + + const imported = await importSessions( + [ + session({ id: 'local', name: 'session-1', createdAt: 1, summary: 'remote copy' }), + session({ id: 'kept', name: 'session-1', createdAt: 2, summary: 'remote copy' }), + session({ id: 'new', name: 'session-1', createdAt: 3, workspace_id: 'ws' }) + ], + user.email + ) + expect(imported).toEqual(['new']) + await vi.waitFor(() => + expect(sessionState.sessions.map((s) => [s.id, s.name])).toEqual([ + ['new', 'session-6'], + ['kept', 'session-5'] + ]) + ) + expect(sessionState.sessions.find((s) => s.id === 'kept')?.summary).toBe('kept') + + // The wrong user's name gets nothing written. + expect(await importSessions([session({ id: 'other', createdAt: 4 })], 'nobody@x')).toEqual([]) + }) }) diff --git a/frontend/src/lib/components/workspaceSettings/AISettings.svelte b/frontend/src/lib/components/workspaceSettings/AISettings.svelte index ae9d2640b0..7e8b9cc76e 100644 --- a/frontend/src/lib/components/workspaceSettings/AISettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/AISettings.svelte @@ -28,6 +28,7 @@ import ModelPricing from './ModelPricing.svelte' import AiUsagePanel from './AiUsagePanel.svelte' import { setCopilotInfo } from '$lib/aiStore' + import { backupSettingsChanged } from '$lib/components/sessions/sessionMirror.svelte' import AIPromptsModal from '../settings/AIPromptsModal.svelte' import { Settings } from 'lucide-svelte' import { untrack } from 'svelte' @@ -79,6 +80,7 @@ let usingOpenaiClientCredentialsOauth = $state(false) let workspaceOverrideEditorOpened = $state(false) let copilotDisabled = $state(false) + let sessionsStorageDisabled = $state(false) // --- Initial state for dirty tracking --- let initialAiProviders: Exclude = $state({}) @@ -90,6 +92,7 @@ let initialModelPricing: Record = $state({}) let initialPrompts: Record = $state({}) let initialCopilotDisabled = $state(false) + let initialSessionsStorageDisabled = $state(false) let lastLoadedConfigKey = $state(undefined) function clone(v: T): T { @@ -118,6 +121,7 @@ maxTokensPerModel = clone(config?.max_tokens_per_model ?? {}) modelPricing = clone(config?.model_pricing ?? {}) copilotDisabled = config?.copilot_disabled === true + sessionsStorageDisabled = config?.sessions_storage_disabled === true for (const mode of ['edit', 'fix', 'gen']) { if (!(mode in customPrompts)) { customPrompts[mode] = '' @@ -135,6 +139,7 @@ initialModelPricing = clone(modelPricing) initialPrompts = clone(customPrompts) initialCopilotDisabled = copilotDisabled + initialSessionsStorageDisabled = sessionsStorageDisabled } export function loadFromConfig(config: AIConfig | undefined) { @@ -151,6 +156,7 @@ maxTokensPerModel = clone(initialMaxTokensPerModel) modelPricing = clone(initialModelPricing) copilotDisabled = initialCopilotDisabled + sessionsStorageDisabled = initialSessionsStorageDisabled } $effect(() => { @@ -186,7 +192,8 @@ JSON.stringify(customPrompts) !== JSON.stringify(initialCustomPrompts) || JSON.stringify(maxTokensPerModel) !== JSON.stringify(initialMaxTokensPerModel) || JSON.stringify(modelPricing) !== JSON.stringify(initialModelPricing) || - copilotDisabled !== initialCopilotDisabled + copilotDisabled !== initialCopilotDisabled || + sessionsStorageDisabled !== initialSessionsStorageDisabled ) $effect(() => { @@ -291,8 +298,9 @@ .filter(([_, prompt]) => prompt.trim().length > 0) .reduce((acc, [mode, prompt]) => ({ ...acc, [mode]: prompt }), {}) - // The flag is the one thing a workspace on instance defaults still stores of its own. + // The flags are what a workspace on instance defaults still stores of its own. const copilot_disabled = copilotDisabled ? true : undefined + const sessions_storage_disabled = sessionsStorageDisabled ? true : undefined return Object.keys(aiProviders ?? {}).length > 0 ? { providers: aiProviders, @@ -303,9 +311,10 @@ max_tokens_per_model: Object.keys(maxTokensPerModel).length > 0 ? maxTokensPerModel : undefined, model_pricing: Object.keys(modelPricing).length > 0 ? modelPricing : undefined, - copilot_disabled + copilot_disabled, + sessions_storage_disabled } - : { copilot_disabled } + : { copilot_disabled, sessions_storage_disabled } } function isSaveDisabled(): boolean { @@ -332,6 +341,7 @@ async function editCopilotConfig(): Promise { const config = buildConfig() + const backupsToggled = sessionsStorageDisabled !== initialSessionsStorageDisabled let settingsState: GetCopilotSettingsStateResponse | undefined if (customSave) { @@ -348,6 +358,9 @@ instance_ai_summary: response.instance_ai_summary } sendUserToast('AI settings updated') + // This page's session backups follow the switch at once, rather than at the + // next page load. + if (backupsToggled) backupSettingsChanged(effectiveWorkspace) } storeInitialState() // Hand the parent what was persisted: it owns `initialConfig`, and this component is @@ -646,6 +659,18 @@ options={{ right: 'Hide AI sessions in this workspace' }} /> + + { + sessionsStorageDisabled = e.detail + }} + options={{ right: 'Do not back AI sessions up to the workspace storage' }} + /> + {/if} diff --git a/frontend/src/lib/userScopedDb.ts b/frontend/src/lib/userScopedDb.ts index 71d80a369f..d1283b2c73 100644 --- a/frontend/src/lib/userScopedDb.ts +++ b/frontend/src/lib/userScopedDb.ts @@ -1,4 +1,11 @@ -import { openDB as idbOpenDB, deleteDB as idbDeleteDB, type DBSchema, type IDBPDatabase } from 'idb' +import { + openDB as idbOpenDB, + deleteDB as idbDeleteDB, + type DBSchema, + type IDBPDatabase, + type IDBPTransaction, + type StoreNames +} from 'idb' import { scopedKey } from '$lib/userScopedStorage' // Per-user IndexedDB lifecycle, shared by the session list and the copilot @@ -21,7 +28,12 @@ export interface UserScopedDbMigrateDeps { export interface UserScopedDbOptions { version: number - upgrade: (db: IDBPDatabase) => void + // The version-change transaction is the only way to add an index to a store that + // already exists; a store being created gets it from the store handle instead. + upgrade: ( + db: IDBPDatabase, + tx: IDBPTransaction[], 'versionchange'> + ) => void // Invoked once per scoped name right after a successful open. The fn owns its // own "already migrated / not applicable" gate (e.g. checking a store's // count) — claim-then-delete legacy data lives here. @@ -100,11 +112,11 @@ export function userScopedDb( try { let handle: IDBPDatabase | undefined const db = await openDB(name, opts.version, { - upgrade(database) { + upgrade(database, _oldVersion, _newVersion, transaction) { // The version-change transaction is ours: nothing is queued ahead of this // open any more, and what remains is our own upgrade running. stopWaiting() - opts.upgrade(database) + opts.upgrade(database, transaction) }, // Another tab is opening this database at a higher version, which our open // connection would block indefinitely. Let go so their upgrade lands; this diff --git a/frontend/src/lib/userScopedStorage.ts b/frontend/src/lib/userScopedStorage.ts index 905d609b61..e3ee1d842d 100644 --- a/frontend/src/lib/userScopedStorage.ts +++ b/frontend/src/lib/userScopedStorage.ts @@ -52,7 +52,19 @@ export function getCurrentUserEmail(): string | undefined { // treat that as "do not read/write" so we never touch a browser-global key. export function scopedKey(base: string): string | undefined { if (!currentEmail) return undefined - return `${base}::${currentEmail}` + return scopedKeyFor(base, currentEmail) +} + +// The key a base name has for a given user, for work that captured its user up front and +// must not follow an in-place account switch (the session backup flush). +export function scopedKeyFor(base: string, email: string): string { + return `${base}::${email}` +} + +// The email a scoped key or database name was built for, so a write that landed in a +// store can name the user it belongs to even after the current user changed. +export function emailOfScopedKey(base: string, key: string): string | undefined { + return key.startsWith(`${base}::`) ? key.slice(base.length + 2) : undefined } // Register a callback invoked whenever the scoping email changes. Fired once diff --git a/frontend/src/routes/(root)/(logged)/+layout.svelte b/frontend/src/routes/(root)/(logged)/+layout.svelte index 4447a8d4b8..225b9443d3 100644 --- a/frontend/src/routes/(root)/(logged)/+layout.svelte +++ b/frontend/src/routes/(root)/(logged)/+layout.svelte @@ -92,6 +92,7 @@ import { parsePreviewItemRoute } from '$lib/components/sessions/previewPaths' import { rememberNavRoute } from '$lib/components/sessions/sessionSwitch.svelte' import { sessionState } from '$lib/components/sessions/sessionState.svelte' + import { restoreSessionBackups } from '$lib/components/sessions/sessionMirror.svelte' import { currentWorkspaceRootId } from '$lib/components/sessions/sessionScope.svelte' import WorkspaceScopeHeader from '$lib/components/sidebar/WorkspaceScopeHeader.svelte' import { DEFAULT_HUB_BASE_URL } from '$lib/hub' @@ -725,6 +726,16 @@ $workspaceStore untrack(() => updateUserStore($workspaceStore)) }) + // Bring back the AI sessions this browser lacks for the workspace family in view, once + // the local list is known (so nothing it has is fetched again) and the memberships have + // resolved (the family is derived from them). + $effect(() => { + const ws = $workspaceStore + const ready = sessionState.hydrated && $usersWorkspaceStore !== undefined + if (globalAiEnabled && ready && ws && !$userStore?.operator) { + untrack(() => restoreSessionBackups(ws)) + } + }) // While a fork is reachable, mirror its parent linkage to localStorage so a // later reload landing on a now-deleted fork can return to the parent (see // forkParentMemory + the deleted-fork recovery in the root layout). From 1c17b3c8dbc43b26adc0a2d4e738cacff8a54e8d Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 15 Sep 2026 22:18:37 +0200 Subject: [PATCH 29/44] test: pin unlisting on a failed multi-object ai session push (#11150) * test: pin that a failed multi-object incremental push leaves the session unlisted Co-Authored-By: Claude Fable 5.1 * test: pick the newest backup generation in the ai sessions test helper Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Fable 5.1 --- backend/tests/ai_sessions.rs | 107 ++++++++++++++++++++++++++++++++++- 1 file changed, 104 insertions(+), 3 deletions(-) diff --git a/backend/tests/ai_sessions.rs b/backend/tests/ai_sessions.rs index f76385d4e1..340a1380da 100644 --- a/backend/tests/ai_sessions.rs +++ b/backend/tests/ai_sessions.rs @@ -87,7 +87,8 @@ async fn rotate(base: &str, key: &str) -> anyhow::Result<()> { } /// The user's prefix on disk, `windmill_ai_sessions/{w_id}/g{generation}/{email hash}`, -/// under whichever generation is current. +/// under the newest generation: deleting an older generation's objects leaves its +/// directories behind, and `read_dir` order differs across filesystems. fn user_root(storage_dir: &std::path::Path, email: &str) -> std::path::PathBuf { let workspace = storage_dir.join("windmill_ai_sessions/test-workspace"); let hash = calculate_hash(email); @@ -96,8 +97,18 @@ fn user_root(storage_dir: &std::path::Path, email: &str) -> std::path::PathBuf { .into_iter() .flatten() .flatten() - .map(|entry| entry.path().join(&hash)) - .find(|path| path.exists()) + .filter_map(|entry| { + let generation: i64 = entry + .file_name() + .to_str()? + .strip_prefix('g')? + .parse() + .ok()?; + Some((generation, entry.path().join(&hash))) + }) + .filter(|(_, path)| path.exists()) + .max_by_key(|(generation, _)| *generation) + .map(|(_, path)| path) .expect("the user has backups under the current key") } @@ -945,6 +956,96 @@ async fn test_backups_round_trip_encrypted_and_scoped_to_the_user( .await?; assert_eq!(resp.status(), 200); + // An incremental part changing more than one object unlists the session before its + // writes, so one write failing after another landed leaves it absent rather than listed + // as a mix of old and new pieces. A directory planted at `artifacts.json` fails that write. + let s10_head = + json!({ "id": "s10", "workspace_id": "test-workspace", "createdAt": 10, "chatId": "c1" }); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s10", "whole": true, "head": s10_head, "chats": s9_chats(&["c1"]), "artifacts": { "items": ["a1"] } }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let s10_dir = user_root(storage_dir.path(), "test@windmill.dev").join("sessions/s10"); + let artifacts_path = s10_dir.join("artifacts.json"); + std::fs::remove_file(&artifacts_path)?; + std::fs::create_dir(&artifacts_path)?; + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s10", "chats": s9_chats(&["c2"]), "artifacts": { "items": ["a2"] } }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let answer: Value = resp.json().await?; + assert!( + answer["results"][0]["error"].is_string(), + "the artifacts write must fail: {answer}" + ); + assert!(answer["results"][0]["needs_whole"].is_null()); + assert!( + s10_dir.join("chats/c2.json").is_file(), + "the chat landed before the artifacts failed" + ); + let s10_listed = |listing: Value| { + listing["sessions"] + .as_array() + .unwrap() + .iter() + .any(|s| s["id"] == "s10") + }; + assert!(!s10_listed(list(&base, "SECRET_TOKEN").await?)); + assert_eq!( + pull(&base, "SECRET_TOKEN", &["s10"]).await?["sessions"], + json!([]) + ); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s10", "chats": s9_chats(&["c3"]) }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + let answer: Value = resp.json().await?; + assert_eq!(answer["results"][0]["needs_whole"], true); + assert!(!s10_listed(list(&base, "SECRET_TOKEN").await?)); + std::fs::remove_dir(&artifacts_path)?; + let resp = push( + &base, + "SECRET_TOKEN", + json!({ + "owner": "test@windmill.dev", + "sessions": [{ "id": "s10", "whole": true, "head": s10_head, "chats": s9_chats(&["c1", "c2", "c3"]), "artifacts": { "items": ["a2"] } }] + }), + ) + .await?; + assert_eq!(resp.status(), 200); + assert!(s10_listed(list(&base, "SECRET_TOKEN").await?)); + let pulled = pull(&base, "SECRET_TOKEN", &["s10"]).await?; + assert_eq!( + pulled["sessions"][0]["artifacts"], + json!({ "items": ["a2"] }) + ); + assert_eq!(pulled_chats(pulled), vec!["c1", "c2", "c3"]); + let resp = push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "removed": ["s10"] }), + ) + .await?; + assert_eq!(resp.status(), 200); + // Removal empties both prefixes. let resp = push( &base, From f082fddf419218e880371f28abe477c64bd7394a Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 15 Sep 2026 23:09:45 +0200 Subject: [PATCH 30/44] [ee] feat: fall back to instance storage for AI session backups (#11153) * feat: instance object store as fallback for AI session backups Co-Authored-By: Claude Fable 5.1 * fix: fence the instance store sweep by generation, name it by location Co-Authored-By: Claude Opus 5 (1M context) * test: pin that an instance store location tells endpoints apart Co-Authored-By: Claude Opus 5 (1M context) * fix: show the instance storage fallback setting on while it is unset Co-Authored-By: Claude Opus 5 (1M context) * fix: check the generation fence queries at compile time Co-Authored-By: Claude Opus 5 (1M context) * fix: stop the instance storage fallback once the plan is Pro Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Fable 5.1 --- ...7f9ed1c0f4b4d1f07789b40945cfb8a3a7b39.json | 14 ++ ...e67c90e86d35dd5f74b1366f802dbb951ef9a.json | 22 ++ ...cb6a994e969d915d50afbf0cb8547e6decf11.json | 22 ++ backend/ee-repo-ref.txt | 2 +- backend/tests/ai_sessions.rs | 226 ++++++++++++++++- .../src/ai_session_backups.rs | 231 ++++++++++++++---- .../windmill-api-workspaces/src/workspaces.rs | 26 ++ backend/windmill-api/openapi.yaml | 7 + backend/windmill-api/src/ai_sessions.rs | 85 +++++-- .../windmill-common/src/global_settings.rs | 5 + .../windmill-common/src/instance_config.rs | 2 + backend/windmill-object-store/src/lib.rs | 155 +++++++++--- docs/ai-session-backups.md | 52 +++- .../lib/components/InstanceSettings.svelte | 8 +- .../src/lib/components/instanceSettings.ts | 11 + .../sessions/sessionMirror.svelte.ts | 31 ++- .../components/sessions/sessionMirror.test.ts | 71 ++++++ .../components/sessions/sessionMirrorPlan.ts | 34 ++- .../workspaceSettings/AISettings.svelte | 2 +- 19 files changed, 871 insertions(+), 135 deletions(-) create mode 100644 backend/.sqlx/query-c2f3492c2d80f5c6d157d8c1dab7f9ed1c0f4b4d1f07789b40945cfb8a3a7b39.json create mode 100644 backend/.sqlx/query-dfa82a3f291cdc8f05cc4af6114e67c90e86d35dd5f74b1366f802dbb951ef9a.json create mode 100644 backend/.sqlx/query-ea7bc2e5f53144ca23f8b3dea71cb6a994e969d915d50afbf0cb8547e6decf11.json diff --git a/backend/.sqlx/query-c2f3492c2d80f5c6d157d8c1dab7f9ed1c0f4b4d1f07789b40945cfb8a3a7b39.json b/backend/.sqlx/query-c2f3492c2d80f5c6d157d8c1dab7f9ed1c0f4b4d1f07789b40945cfb8a3a7b39.json new file mode 100644 index 0000000000..9724a99b18 --- /dev/null +++ b/backend/.sqlx/query-c2f3492c2d80f5c6d157d8c1dab7f9ed1c0f4b4d1f07789b40945cfb8a3a7b39.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings SET ai_sessions_backup_generation = ai_sessions_backup_generation + 1 WHERE workspace_id = $1 AND large_file_storage IS NULL", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "c2f3492c2d80f5c6d157d8c1dab7f9ed1c0f4b4d1f07789b40945cfb8a3a7b39" +} diff --git a/backend/.sqlx/query-dfa82a3f291cdc8f05cc4af6114e67c90e86d35dd5f74b1366f802dbb951ef9a.json b/backend/.sqlx/query-dfa82a3f291cdc8f05cc4af6114e67c90e86d35dd5f74b1366f802dbb951ef9a.json new file mode 100644 index 0000000000..0b17793580 --- /dev/null +++ b/backend/.sqlx/query-dfa82a3f291cdc8f05cc4af6114e67c90e86d35dd5f74b1366f802dbb951ef9a.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT large_file_storage IS NOT NULL AS \"has_storage!\" FROM workspace_settings WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "has_storage!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "dfa82a3f291cdc8f05cc4af6114e67c90e86d35dd5f74b1366f802dbb951ef9a" +} diff --git a/backend/.sqlx/query-ea7bc2e5f53144ca23f8b3dea71cb6a994e969d915d50afbf0cb8547e6decf11.json b/backend/.sqlx/query-ea7bc2e5f53144ca23f8b3dea71cb6a994e969d915d50afbf0cb8547e6decf11.json new file mode 100644 index 0000000000..a344e64392 --- /dev/null +++ b/backend/.sqlx/query-ea7bc2e5f53144ca23f8b3dea71cb6a994e969d915d50afbf0cb8547e6decf11.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT ai_sessions_backup_generation FROM workspace_settings WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "ai_sessions_backup_generation", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "ea7bc2e5f53144ca23f8b3dea71cb6a994e969d915d50afbf0cb8547e6decf11" +} diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 7acd297dac..5d28f73b5a 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -1c1dab33563c4907aff8b0da825fb66db60af82a +93433d7c9dc34f2c0f56a5297d3453aabd2f9472 \ No newline at end of file diff --git a/backend/tests/ai_sessions.rs b/backend/tests/ai_sessions.rs index 340a1380da..3eb28cfae8 100644 --- a/backend/tests/ai_sessions.rs +++ b/backend/tests/ai_sessions.rs @@ -34,6 +34,54 @@ async fn configure_primary_lfs(db: &Pool, root_path: &str) -> anyhow:: Ok(()) } +/// Configures the primary storage through the route, which is what sweeps the workspace's +/// backups out of the instance store. +async fn configure_primary_lfs_via_route(base: &str, root_path: &str) -> anyhow::Result<()> { + let resp = authed( + client().post(format!("{base}/workspaces/edit_large_file_storage_config")), + "SECRET_TOKEN", + ) + .json(&json!({ "large_file_storage": { + "type": "FilesystemStorage", + "root_path": root_path, + "public_resource": false, + "advanced_permissions": null, + "secondary_storage": {} + }})) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + Ok(()) +} + +/// The instance setting allowing the instance store to stand in for a workspace without +/// storage: `None` leaves it unset, which is on. +async fn set_instance_fallback(db: &Pool, on: Option) -> anyhow::Result<()> { + sqlx::query("DELETE FROM global_settings WHERE name = 'ai_sessions_instance_storage_fallback'") + .execute(db) + .await?; + if let Some(on) = on { + sqlx::query( + "INSERT INTO global_settings (name, value) VALUES ('ai_sessions_instance_storage_fallback', $1)", + ) + .bind(json!(on)) + .execute(db) + .await?; + } + Ok(()) +} + +/// Polls until nothing is under the directory, for a deletion that runs off the request. +async fn wait_until_empty(dir: &std::path::Path, what: &str) { + for _ in 0..100 { + if files_under(dir).is_empty() { + return; + } + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + panic!("{what}: objects left under {}", dir.display()); +} + async fn list(base: &str, token: &str) -> anyhow::Result { let resp = authed(client().get(format!("{base}/ai/sessions/list")), token) .send() @@ -143,7 +191,9 @@ async fn test_backups_round_trip_encrypted_and_scoped_to_the_user( server.addr.port() ); - // No storage configured: the browser is told to stop trying. + // No storage configured, and the instance store (another test of this process may + // have loaded one) not allowed to stand in: the browser is told to stop trying. + set_instance_fallback(&db, Some(false)).await?; let listing = list(&base, "SECRET_TOKEN").await?; assert_eq!(listing["enabled"], false); assert_eq!(listing["sessions"], json!([])); @@ -1184,3 +1234,177 @@ async fn test_backup_writes_are_refused_for_the_wrong_owner_token_or_id( assert!(files_under(storage_dir.path()).is_empty()); Ok(()) } + +/// Puts the process-wide instance store back to none, even when an assertion fails. +struct ResetInstanceStore; +impl Drop for ResetInstanceStore { + fn drop(&mut self) { + if let Ok(mut store) = windmill_object_store::OBJECT_STORE_SETTINGS.try_write() { + *store = None; + } + } +} + +/// Puts the process-wide license key id back to none, an Enterprise plan in this build, +/// even when an assertion fails. +struct ResetLicensePlan; +impl Drop for ResetLicensePlan { + fn drop(&mut self) { + windmill_common::ee::LICENSE_KEY_ID.store(std::sync::Arc::new(String::new())); + } +} + +/// A workspace without storage of its own backs up to the instance object store, every +/// answer saying so (`fallback`); a storage of its own, once configured, answers instead, +/// under a generation past everything the workspace left in the instance store, which the +/// change deletes; a plan switched to Pro stops the fallback with the store still loaded. +#[sqlx::test(fixtures("base"))] +async fn test_backups_fall_back_to_the_instance_storage(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let base = format!( + "http://localhost:{}/api/w/test-workspace", + server.addr.port() + ); + + // The instance store, built from its settings as `reload_object_store_setting` does. + let instance_dir = tempfile::tempdir()?; + let instance_root = instance_dir.path().to_string_lossy().to_string(); + *windmill_object_store::OBJECT_STORE_SETTINGS.write().await = Some( + windmill_object_store::build_object_store_from_settings( + windmill_object_store::ObjectSettings::Filesystem( + windmill_object_store::FilesystemSettings { root_path: instance_root.clone() }, + ), + None, + ) + .await?, + ); + let _reset = ResetInstanceStore; + let in_instance = instance_dir + .path() + .join("windmill_ai_sessions/test-workspace"); + + // Turned off by the instance setting: the browser is told to stop trying. + set_instance_fallback(&db, Some(false)).await?; + assert_eq!(list(&base, "SECRET_TOKEN").await?["enabled"], false); + set_instance_fallback(&db, None).await?; + + // On, as it is unless turned off: the backups land in the instance store, under the + // workspace's prefix, and every answer says which kind of store it came from. + let head = + json!({ "id": "s1", "workspace_id": "test-workspace", "createdAt": 1, "chatId": "c1" }); + let entry = json!({ "id": "s1", "whole": true, "head": head, "chats": [{ "id": "c1", "record": { "id": "c1" } }] }); + let push_whole = || { + push( + &base, + "SECRET_TOKEN", + json!({ "owner": "test@windmill.dev", "sessions": [entry.clone()] }), + ) + }; + let resp = push_whole().await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let pushed: Value = resp.json().await?; + assert_eq!(pushed["fallback"], true); + assert_eq!(pushed["results"], json!([{ "id": "s1" }])); + let listing = list(&base, "SECRET_TOKEN").await?; + assert_eq!(listing["enabled"], true); + assert_eq!(listing["fallback"], true); + assert_eq!(listing["sessions"][0]["id"], "s1"); + let fallback_storage_id = listing["storage_id"].clone(); + let pulled = pull(&base, "SECRET_TOKEN", &["s1"]).await?; + assert_eq!(pulled["fallback"], true); + assert_eq!(pulled["sessions"][0]["head"], head); + assert!(!files_under(&in_instance).is_empty()); + + // The workspace's storage usage counts them, under a name of their own. + let resp = authed( + client().get(format!("{base}/job_helpers/storage_usage")), + "SECRET_TOKEN", + ) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let usage: Value = resp.json().await?; + let fallback_usage = usage["storages"] + .as_array() + .unwrap() + .iter() + .find(|s| s["storage"] == "_ai_sessions_fallback_") + .unwrap_or_else(|| panic!("no fallback usage in {usage}")); + assert!(fallback_usage["bytes"].as_i64().unwrap() > 0); + + // A key rotation sweeps the older generation out of the instance store too. + rotate(&base, &"c".repeat(64)).await?; + wait_until_empty(&in_instance, "a rotation on the instance store").await; + assert_eq!(list(&base, "SECRET_TOKEN").await?["sessions"], json!([])); + assert_eq!(push_whole().await?.status(), 200); + assert!(!files_under(&in_instance).is_empty()); + + // A storage of its own answers instead, under a generation the configuration moved past + // everything the workspace left in the instance store: nothing there is read again, + // whichever store a later return to the fallback finds, and it is deleted. + let before = list(&base, "SECRET_TOKEN").await?; + let storage_dir = tempfile::tempdir()?; + configure_primary_lfs_via_route(&base, &storage_dir.path().to_string_lossy()).await?; + wait_until_empty(&in_instance, "configuring a workspace storage").await; + let listing = list(&base, "SECRET_TOKEN").await?; + assert_eq!(listing["enabled"], true); + assert!(listing.get("fallback").is_none(), "{listing}"); + assert_ne!(listing["storage_id"], fallback_storage_id); + assert_eq!( + listing["backup_generation"].as_i64(), + before["backup_generation"].as_i64().map(|g| g + 1), + "configuring a storage over the fallback must move the generation on" + ); + assert_eq!(listing["sessions"], json!([])); + assert_eq!(push_whole().await?.status(), 200); + assert!(!files_under(storage_dir.path()).is_empty()); + assert!(files_under(&in_instance).is_empty()); + let resp = authed( + client().get(format!("{base}/job_helpers/storage_usage?refresh=true")), + "SECRET_TOKEN", + ) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let usage: Value = resp.json().await?; + assert!( + !usage.to_string().contains("_ai_sessions_fallback_"), + "nothing is counted in the instance store for a workspace with storage: {usage}" + ); + + // Pointed at the instance store's own bucket, a storage of its own keeps its live + // backups there under the current generation, which no storage change deletes. + configure_primary_lfs_via_route(&base, &instance_root).await?; + assert_eq!(push_whole().await?.status(), 200); + assert!(!files_under(&in_instance).is_empty()); + let same = list(&base, "SECRET_TOKEN").await?; + configure_primary_lfs_via_route(&base, &instance_root).await?; + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + assert!(!files_under(&in_instance).is_empty()); + let listing = list(&base, "SECRET_TOKEN").await?; + assert!(listing.get("fallback").is_none(), "{listing}"); + assert_eq!(listing["backup_generation"], same["backup_generation"]); + assert_eq!(listing["sessions"][0]["id"], "s1"); + + // Back to no storage of its own, the fallback answers; a plan switched to Pro while the + // instance store stays loaded stops it at once, for the listing and the push alike. + let resp = authed( + client().post(format!("{base}/workspaces/edit_large_file_storage_config")), + "SECRET_TOKEN", + ) + .json(&json!({ "large_file_storage": null })) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + assert_eq!(list(&base, "SECRET_TOKEN").await?["fallback"], true); + let _enterprise_again = ResetLicensePlan; + windmill_common::ee::LICENSE_KEY_ID.store(std::sync::Arc::new("test_pro".to_string())); + assert_eq!(list(&base, "SECRET_TOKEN").await?["enabled"], false); + let resp = push_whole().await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let pushed: Value = resp.json().await?; + assert_eq!(pushed["enabled"], false, "{pushed}"); + + Ok(()) +} diff --git a/backend/windmill-api-workspaces/src/ai_session_backups.rs b/backend/windmill-api-workspaces/src/ai_session_backups.rs index bbdbfbc8c9..46c91d5354 100644 --- a/backend/windmill-api-workspaces/src/ai_session_backups.rs +++ b/backend/windmill-api-workspaces/src/ai_session_backups.rs @@ -1,5 +1,6 @@ -//! What the workspace key rotation and the AI session backup routes -//! (`windmill-api/src/ai_sessions.rs`) share about the backups in the workspace storage. +//! What the workspace key rotation, the workspace storage settings and the AI session backup +//! routes (`windmill-api/src/ai_sessions.rs`) share about the backups: the store they live +//! in, and what a rotation or a storage change deletes. //! //! The backups are ciphertext under the workspace key and live under a prefix named by a //! generation the rotation bumps (`workspace_settings.ai_sessions_backup_generation`) in the @@ -13,17 +14,29 @@ //! one to read with and nothing to rewrite in place. A generation is never reused, so no //! deletion, however late, can touch live objects; a rotation that fails before its commit //! bumps nothing and deletes nothing; two rotations racing serialize on the key row. +//! +//! A workspace without storage of its own keeps its backups in the instance object store +//! instead, under the same layout and key, while `ai_sessions_instance_storage_fallback` +//! allows it. Configuring a storage for such a workspace bumps the generation in the +//! transaction that sets it, so everything the workspace left in any instance store sits +//! under a generation the routes never read again: a later return to the instance store, +//! whichever it is by then, starts from a newer one. That is what lets a storage change +//! delete the older generations from the instance store without fencing against what +//! happens next, and a browser retire a removal owed to an instance store once the +//! workspace's own storage answered. use std::sync::Arc; -use futures::TryStreamExt; +use futures::{StreamExt, TryStreamExt}; use windmill_common::error::{Error, Result}; use windmill_common::utils::calculate_hash; use windmill_common::DB; use windmill_object_store::object_store_reexports::{ ObjectStore, ObjectStoreError, Path as ObjectPath, }; -use windmill_object_store::{object_store_error_to_error, ObjectStoreResource}; +use windmill_object_store::{ + object_store_error_to_error, object_store_location, ObjectStoreResource, +}; use windmill_types::s3::LargeFileStorage; /// The root of every AI session backup key in a workspace's storage. @@ -31,6 +44,9 @@ pub const ROOT: &str = "windmill_ai_sessions"; /// The push body cap: no object written through the routes is larger. One that is was /// planted by whoever holds the bucket's credentials, and is left unread. pub const MAX_OBJECT_BYTES: usize = 32 * 1024 * 1024; +/// The storage name the workspace's backups in the instance store count under in its +/// storage usage, next to `_default_` and the secondary storages. +pub const FALLBACK_STORAGE: &str = "_ai_sessions_fallback_"; const IO_CONCURRENCY: usize = 8; @@ -39,34 +55,76 @@ pub fn generation_prefix(w_id: &str, generation: i64) -> String { format!("{ROOT}/{w_id}/g{generation}") } +/// The prefix of everything the workspace ever backed up, whatever the generation. +fn workspace_prefix(w_id: &str) -> ObjectPath { + ObjectPath::from(format!("{ROOT}/{w_id}")) +} + /// Names the storage the backups are in, by what locates its objects (endpoint, region, /// bucket; never the credentials, which rotate), so a browser tells that its sync state was /// recorded against another storage; the generation, answered alongside, tells it a /// rotation happened in this one. pub fn storage_id(resource: &ObjectStoreResource) -> String { - let location = match resource { - ObjectStoreResource::S3(s) => format!( - "s3:{}:{}:{}:{}", - s.endpoint, - s.port.unwrap_or_default(), - s.region, - s.bucket - ), - ObjectStoreResource::Azure(a) => format!( - "azure:{}:{}:{}", - a.endpoint.as_deref().unwrap_or_default(), - a.account_name, - a.container_name - ), - ObjectStoreResource::Gcs(g) => format!("gcs:{}", g.bucket), - ObjectStoreResource::Filesystem(f) => format!("fs:{}", f.root_path), - }; - calculate_hash(&location)[..16].to_string() + calculate_hash(&object_store_location(resource))[..16].to_string() } -/// The workspace's primary storage, resolved without a caller: a rotation runs the -/// deletion off its own request. -async fn primary_store(db: &DB, w_id: &str) -> Result>> { +/// Where a workspace's backups live: its primary storage, or the instance object store +/// standing in for it. +pub struct BackupStore { + pub store: Arc, + pub storage_id: String, + pub fallback: bool, +} + +/// The instance object store, for a workspace without storage of its own: loaded from +/// settings that say where its objects are, and not turned off by +/// `ai_sessions_instance_storage_fallback`, which is on unless set to false. Named like a +/// workspace storage, by that location, in a namespace of its own. Never on the Pro plan, +/// checked on every call: a store loaded before a switch to Pro stays loaded. Never in a +/// build without `private`, which has neither workspace storage nor the quota the fallback +/// counts toward. +/// +/// Authorizes nothing, and the store reaches every workspace's objects: the caller must have +/// authorized the user for the workspace and keep what it reads and writes under that +/// user's prefix in it, as the backup routes do. +pub async fn fallback_store(db: &DB) -> Result> { + #[cfg(not(feature = "private"))] + { + let _ = db; + Ok(None) + } + #[cfg(feature = "private")] + { + if matches!( + windmill_common::ee_oss::get_license_plan().await, + windmill_common::ee_oss::LicensePlan::Pro + ) { + return Ok(None); + } + let Some((store, Some(location))) = + windmill_object_store::get_object_store_with_location().await + else { + return Ok(None); + }; + let setting = windmill_common::global_settings::load_value_from_global_settings( + db, + windmill_common::global_settings::AI_SESSIONS_INSTANCE_STORAGE_FALLBACK_SETTING, + ) + .await?; + if matches!(setting, Some(serde_json::Value::Bool(false))) { + return Ok(None); + } + Ok(Some(BackupStore { + storage_id: calculate_hash(&format!("instance:{location}"))[..16].to_string(), + store, + fallback: true, + })) + } +} + +/// The workspace's primary storage, resolved without a caller: a rotation runs its deletion +/// off its own request. +async fn primary_store(db: &DB, w_id: &str) -> Result> { let Some(lfs_json) = sqlx::query_scalar!( "SELECT large_file_storage FROM workspace_settings WHERE workspace_id = $1", w_id @@ -91,9 +149,19 @@ async fn primary_store(db: &DB, w_id: &str) -> Result Result> { + if let Some(primary) = primary_store(db, w_id).await? { + return Ok(Some(primary)); + } + fallback_store(db).await } /// The generation an object key sits under, `None` for a key of no generation (an older @@ -107,40 +175,41 @@ fn generation_of(w_id: &str, key: &ObjectPath) -> Option { .ok() } -/// Deletes, off the request and as the listing streams, every object of the workspace's -/// backups from a generation older than `current`, once the rotation that made `current` -/// the generation has committed: nothing writes there any more but a push that resolved its -/// prefix before the commit, junk the browser's next push of that session rewrites under the -/// current prefix, as is anything a deletion cut short left behind. For the rotation route, -/// which authorized its caller as a superadmin. +/// Deletes, as the listing streams, every object of the workspace's backups in the store +/// from a generation older than `current`. +async fn delete_older(store: &Arc, w_id: &str, current: i64) -> Result<()> { + store + .list(Some(&workspace_prefix(w_id))) + .map_err(object_store_error_to_error) + .try_for_each_concurrent(IO_CONCURRENCY, |meta| async move { + if generation_of(w_id, &meta.location).is_some_and(|g| g >= current) { + return Ok(()); + } + match store.delete(&meta.location).await { + Ok(()) | Err(ObjectStoreError::NotFound { .. }) => Ok(()), + Err(e) => Err(object_store_error_to_error(e)), + } + }) + .await +} + +/// Deletes, off the request, every object of the workspace's backups from a generation +/// older than `current`, once the rotation that made `current` the generation has +/// committed: nothing writes there any more but a push that resolved its prefix before the +/// commit, junk the browser's next push of that session rewrites under the current prefix, +/// as is anything a deletion cut short left behind. For the rotation route, which +/// authorized its caller as a superadmin. pub(crate) fn spawn_delete_older(db: DB, w_id: String, current: i64) { tokio::spawn(async move { - let store = match primary_store(&db, &w_id).await { - Ok(Some(store)) => store, + let store = match workspace_store(&db, &w_id).await { + Ok(Some(store)) => store.store, Ok(None) => return, Err(e) => { tracing::warn!("older AI session backups of {w_id} left in place: {e:#}"); return; } }; - let prefix = ObjectPath::from(format!("{ROOT}/{w_id}")); - let deleted = store - .list(Some(&prefix)) - .map_err(object_store_error_to_error) - .try_for_each_concurrent(IO_CONCURRENCY, |meta| { - let (store, w_id) = (&store, &w_id); - async move { - if generation_of(w_id, &meta.location).is_some_and(|g| g >= current) { - return Ok(()); - } - match store.delete(&meta.location).await { - Ok(()) | Err(ObjectStoreError::NotFound { .. }) => Ok(()), - Err(e) => Err(object_store_error_to_error(e)), - } - } - }) - .await; - match deleted { + match delete_older(&store, &w_id, current).await { Ok(()) => { tracing::info!("deleted the AI session backups of {w_id} older than g{current}") } @@ -148,3 +217,57 @@ pub(crate) fn spawn_delete_older(db: DB, w_id: String, current: i64) { } }); } + +/// Deletes, off the request, what the workspace's backups left in the instance store under +/// a generation older than `current`, the one a storage settings change committed. Nothing +/// reads there: the routes use the workspace's own storage, or, back in the instance store, +/// `current` or a newer generation, since configuring a storage over the fallback bumped +/// it. So it runs whatever the storage is now and whatever the setting says (copies from +/// when it was on may be there), and a deletion that is slow, cut short or overtaken by a +/// later change deletes nothing live. For the storage settings route, which authorized its +/// caller as a workspace admin. +pub(crate) fn spawn_delete_fallback(w_id: String, current: i64) { + tokio::spawn(async move { + let Some(instance) = windmill_object_store::get_object_store().await else { + return; + }; + match delete_older(&instance, &w_id, current).await { + Ok(()) => tracing::info!( + "deleted the AI session backups of {w_id} older than g{current} from the instance store" + ), + Err(e) => tracing::warn!( + "deleting the AI session backups of {w_id} from the instance store: {e:#}" + ), + } + }); +} + +/// The bytes of the workspace's backups in the instance store, for its storage usage while +/// it has no storage of its own (once it has one nothing writes there, and the change +/// deleted what was): `None` when it has one, when there is no instance store, or when +/// there is nothing, so no empty usage entry shows up. Whether the setting is on or off, +/// since copies from when it was on may be there. +/// +/// Authorizes nothing: for the storage usage recount, which reports a total for the +/// workspace it was run for and hands out nothing it read. +pub async fn fallback_bytes(db: &DB, w_id: &str) -> Result> { + let has_storage = sqlx::query_scalar!( + r#"SELECT large_file_storage IS NOT NULL AS "has_storage!" FROM workspace_settings WHERE workspace_id = $1"#, + w_id + ) + .fetch_optional(db) + .await? + .unwrap_or(false); + if has_storage { + return Ok(None); + } + let Some(instance) = windmill_object_store::get_object_store().await else { + return Ok(None); + }; + let mut total: i64 = 0; + let mut stream = instance.list(Some(&workspace_prefix(w_id))); + while let Some(meta) = stream.next().await { + total += meta.map_err(object_store_error_to_error)?.size as i64; + } + Ok((total > 0).then_some(total)) +} diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 1a3c24f096..c240948105 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -2093,6 +2093,17 @@ async fn edit_large_file_storage_config( serde_json::to_value::(lfs_config) .map_err(|err| Error::internal_err(err.to_string()))?; + // A workspace whose AI session backups fell back to the instance store leaves it + // here: the generation moves on, so nothing it left in any instance store is read + // again, whichever one a later return to the fallback finds (`ai_session_backups`). + sqlx::query!( + "UPDATE workspace_settings SET ai_sessions_backup_generation = \ + ai_sessions_backup_generation + 1 \ + WHERE workspace_id = $1 AND large_file_storage IS NULL", + &w_id + ) + .execute(&mut *tx) + .await?; sqlx::query!( "UPDATE workspace_settings SET large_file_storage = $1 WHERE workspace_id = $2", serialized_lfs_config, @@ -2108,8 +2119,23 @@ async fn edit_large_file_storage_config( .execute(&mut *tx) .await?; } + let backups_generation = sqlx::query_scalar!( + "SELECT ai_sessions_backup_generation FROM workspace_settings WHERE workspace_id = $1", + &w_id + ) + .fetch_optional(&mut *tx) + .await?; tx.commit().await?; + // Read by nothing any more, whatever the storage is now: what the AI session backups + // left in the instance store under a generation older than the one just committed. + #[cfg(feature = "parquet")] + if let Some(generation) = backups_generation { + crate::ai_session_backups::spawn_delete_fallback(w_id.clone(), generation); + } + #[cfg(not(feature = "parquet"))] + let _ = backups_generation; + // Trigger git sync for large file storage changes handle_deployment_metadata( &authed.email, diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 9f742a129b..63e8dd46ea 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -13222,6 +13222,9 @@ paths: backup_generation: type: integer description: bumped by every workspace key rotation; sync state recorded under another one is void + fallback: + type: boolean + description: the storage answered from is the instance object store, standing in for a workspace without storage of its own; a removal owed to it is retired by any answer from the workspace's own storage sessions: type: array description: the newest 500 at most @@ -13273,6 +13276,8 @@ paths: type: string backup_generation: type: integer + fallback: + type: boolean sessions: type: array items: @@ -13327,6 +13332,8 @@ paths: type: string backup_generation: type: integer + fallback: + type: boolean results: type: array items: diff --git a/backend/windmill-api/src/ai_sessions.rs b/backend/windmill-api/src/ai_sessions.rs index fca9c35fca..776b3a0561 100644 --- a/backend/windmill-api/src/ai_sessions.rs +++ b/backend/windmill-api/src/ai_sessions.rs @@ -30,7 +30,7 @@ use serde_json::value::RawValue; use std::sync::Arc; use windmill_api_auth::is_effectively_unscoped; use windmill_api_workspaces::ai_session_backups::{ - generation_prefix, storage_id, MAX_OBJECT_BYTES, + fallback_store, generation_prefix, storage_id, MAX_OBJECT_BYTES, }; use windmill_common::error::{Error, JsonResult, Result}; use windmill_common::utils::calculate_hash; @@ -101,6 +101,9 @@ struct Backend { /// owed to the storage alone (a rotation deleted the older generation's copy anyway). storage_id: String, generation: i64, + /// The store is the instance object store standing in for a workspace without storage + /// of its own (`ai_session_backups::fallback_store`). + fallback: bool, } impl Backend { @@ -408,34 +411,56 @@ fn require_json_object(kind: &str, raw: &RawValue, max_bytes: usize) -> Result<( Ok(()) } -/// `None` when the workspace has nowhere to keep backups: no primary storage configured, or -/// the admin switched them off. Both read as `enabled: false` so the browser stops trying. +/// `None` when the workspace has nowhere to keep backups: no primary storage configured and +/// no instance store to stand in, or the admin switched them off. Both read as +/// `enabled: false` so the browser stops trying. async fn backend(authed: &ApiAuthed, db: &DB, w_id: &str) -> Result> { - let (disabled, generation) = sqlx::query_as::<_, (Option, i64)>( - "SELECT (ai_config->>'sessions_storage_disabled')::bool, ai_sessions_backup_generation \ - FROM workspace_settings WHERE workspace_id = $1", + let (disabled, generation, has_storage) = sqlx::query_as::<_, (Option, i64, bool)>( + "SELECT (ai_config->>'sessions_storage_disabled')::bool, ai_sessions_backup_generation, \ + large_file_storage IS NOT NULL FROM workspace_settings WHERE workspace_id = $1", ) .bind(w_id) .fetch_optional(db) .await? - .unwrap_or((None, 0)); + .unwrap_or((None, 0, false)); if disabled.unwrap_or(false) { return Ok(None); } - let (_, resource) = - crate::job_helpers_oss::get_workspace_s3_resource(authed, db, None, w_id, None).await?; - let Some(resource) = resource else { - return Ok(None); + // Decided from the row the generation came from: the instance store is written only + // under a generation read while the workspace had no storage of its own, which + // configuring one moves past (`ai_session_backups`). + let (store, storage_id, fallback) = if has_storage { + let (_, resource) = + crate::job_helpers_oss::get_workspace_s3_resource(authed, db, None, w_id, None).await?; + let Some(resource) = resource else { + return Ok(None); + }; + ( + build_object_store_client(&resource).await?, + storage_id(&resource), + false, + ) + } else { + // The instance store stands in, under the same layout and the same key. + match fallback_store(db).await? { + Some(f) => (f.store, f.storage_id, true), + None => return Ok(None), + } }; - let store = build_object_store_client(&resource).await?; let user = calculate_hash(&authed.email); // Keyed per user, not per workspace: anyone who can write the bucket could otherwise copy // another member's ciphertext under their own prefix and have `pull` decrypt it for them. let key = get_workspace_key(w_id, db).await?; let mc = crypt_from_key_with_suffix(&key, &user); - let storage_id = storage_id(&resource); let prefix = format!("{}/{user}", generation_prefix(w_id, generation)); - Ok(Some(Backend { store, mc, prefix, storage_id, generation })) + Ok(Some(Backend { + store, + mc, + prefix, + storage_id, + generation, + fallback, + })) } #[derive(Serialize)] @@ -455,6 +480,12 @@ struct ListResponse { storage_id: Option, #[serde(skip_serializing_if = "Option::is_none")] backup_generation: Option, + /// The storage is the instance store standing in for a workspace without one of its + /// own; a removal owed to it is retired by any answer from the workspace's own storage + /// once it has one (configuring it moved the generation past everything the workspace + /// left in any instance store). + #[serde(skip_serializing_if = "std::ops::Not::not")] + fallback: bool, sessions: Vec, /// The user has more sessions than the answer names. #[serde(skip_serializing_if = "std::ops::Not::not")] @@ -474,6 +505,7 @@ async fn list( enabled: false, storage_id: None, backup_generation: None, + fallback: false, sessions: vec![], truncated: false, })); @@ -526,6 +558,7 @@ async fn list( enabled: true, storage_id: Some(backend.storage_id.clone()), backup_generation: Some(backend.generation), + fallback: backend.fallback, sessions, truncated, })) @@ -594,6 +627,8 @@ struct PullResponse { storage_id: Option, #[serde(skip_serializing_if = "Option::is_none")] backup_generation: Option, + #[serde(skip_serializing_if = "std::ops::Not::not")] + fallback: bool, sessions: Vec, /// Ids that did not fit the response budget; ask for them again. deferred: Vec, @@ -839,6 +874,7 @@ async fn pull( enabled: false, storage_id: None, backup_generation: None, + fallback: false, sessions: vec![], deferred: vec![], })); @@ -861,6 +897,7 @@ async fn pull( enabled: true, storage_id: Some(backend.storage_id), backup_generation: Some(backend.generation), + fallback: backend.fallback, sessions, deferred, })) @@ -952,6 +989,8 @@ struct PushResponse { storage_id: Option, #[serde(skip_serializing_if = "Option::is_none")] backup_generation: Option, + #[serde(skip_serializing_if = "std::ops::Not::not")] + fallback: bool, results: Vec, } @@ -1241,6 +1280,7 @@ async fn push( enabled: false, storage_id: None, backup_generation: None, + fallback: false, results: vec![], })); }; @@ -1294,16 +1334,16 @@ async fn push( results.push(PushResult { id: sid.clone(), error, needs_whole: false }); } // Overwrites and deletes make this an over-count; the periodic recount the quota check - // schedules once usage is stale settles it. + // schedules once usage is stale settles it. Bytes in the instance store count under a + // name of their own, which the recount lists there. #[cfg(not(feature = "enterprise"))] if written > 0 { - crate::job_helpers_oss::bump_storage_usage( - &db, - &w_id, - windmill_object_store::DEFAULT_STORAGE, - written as i64, - ) - .await; + let storage = if backend.fallback { + windmill_api_workspaces::ai_session_backups::FALLBACK_STORAGE + } else { + windmill_object_store::DEFAULT_STORAGE + }; + crate::job_helpers_oss::bump_storage_usage(&db, &w_id, storage, written as i64).await; } #[cfg(feature = "enterprise")] let _ = written; @@ -1311,6 +1351,7 @@ async fn push( enabled: true, storage_id: Some(backend.storage_id), backup_generation: Some(backend.generation), + fallback: backend.fallback, results, })) } diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index 49efa26f60..ef63fc2347 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -84,6 +84,11 @@ pub const SANDBOX_REGISTRY_AUTH_SETTING: &str = "sandbox_registry_auth"; // windmill-worker/src/ssh_executor_ee.rs. pub const SSH_EXECUTION_SETTING: &str = "ssh_execution_enabled"; pub const OBJECT_STORE_CONFIG_SETTING: &str = "object_store_cache_config"; +/// Whether the instance object store stands in for a workspace without storage of its own +/// as the place its members' AI sessions are backed up to. On unless the row says `false`; +/// inert without an instance object store. +pub const AI_SESSIONS_INSTANCE_STORAGE_FALLBACK_SETTING: &str = + "ai_sessions_instance_storage_fallback"; /// Compile a newly deployed script's binary right after its dependency job and push it /// to the instance object store, so the first run does not pay the compile. Inert unless /// instance object storage is configured — without it the binary would only ever land in diff --git a/backend/windmill-common/src/instance_config.rs b/backend/windmill-common/src/instance_config.rs index 1dd3396809..de3fd7684a 100644 --- a/backend/windmill-common/src/instance_config.rs +++ b/backend/windmill-common/src/instance_config.rs @@ -264,6 +264,8 @@ pub struct GlobalSettings { pub disable_hub: Option, #[serde(skip_serializing_if = "Option::is_none")] pub auto_build_binary_on_deploy: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub ai_sessions_instance_storage_fallback: Option, // String settings #[serde(skip_serializing_if = "Option::is_none")] diff --git a/backend/windmill-object-store/src/lib.rs b/backend/windmill-object-store/src/lib.rs index 25dc6ea19c..edc3ea9149 100644 --- a/backend/windmill-object-store/src/lib.rs +++ b/backend/windmill-object-store/src/lib.rs @@ -118,6 +118,10 @@ pub fn object_store_error_to_error(err: object_store::Error) -> error::Error { pub struct ExpirableObjectStore { pub store: Arc, pub refresh: Option, + /// What locates the store's objects ([`object_store_location`]), for a store built from + /// settings. Kept with the store rather than read off the settings again, so a server + /// whose reload is still pending never names one store by another's location. + pub location: Option, } #[cfg(feature = "parquet")] @@ -155,7 +159,7 @@ impl ObjectStoreRefresh { #[cfg(feature = "parquet")] impl From> for ExpirableObjectStore { fn from(store: Arc) -> Self { - Self { store, refresh: None } + Self { store, refresh: None, location: None } } } @@ -197,6 +201,15 @@ static CACHE_OVERRIDE_GENERATION: std::sync::atomic::AtomicU64 = async fn resolve_object_store( settings_lock: &RwLock>, ) -> Option> { + resolve_object_store_with_location(settings_lock) + .await + .map(|(store, _)| store) +} + +#[cfg(feature = "parquet")] +async fn resolve_object_store_with_location( + settings_lock: &RwLock>, +) -> Option<(Arc, Option)> { let settings = settings_lock.read().await; let Some(s) = settings.as_ref() else { return None; @@ -212,18 +225,18 @@ async fn resolve_object_store( // A reload may have installed a different store while the credentials were // being minted; that one reflects newer config, so the refresh is stale. Some(current) if !Arc::ptr_eq(¤t.store, &refreshed_from) => { - Some(current.store.clone()) + Some((current.store.clone(), current.location.clone())) } Some(_) => { - let arc = new_store.store.clone(); + let found = (new_store.store.clone(), new_store.location.clone()); *settings = Some(new_store); - Some(arc) + Some(found) } // Cleared while refreshing. None => None, } } - _ => Some(s.store.clone()), + _ => Some((s.store.clone(), s.location.clone())), } } @@ -232,6 +245,13 @@ pub async fn get_object_store() -> Option> { resolve_object_store(&OBJECT_STORE_SETTINGS).await } +/// The instance object store with what locates its objects ([`object_store_location`]), +/// read together; the location is `None` for a store installed without settings. +#[cfg(feature = "parquet")] +pub async fn get_object_store_with_location() -> Option<(Arc, Option)> { + resolve_object_store_with_location(&OBJECT_STORE_SETTINGS).await +} + /// The store the dependency cache reads and writes: the worker group's override when it has one, /// the instance object store otherwise. Anything the server must also reach goes through /// [`get_object_store`] instead. @@ -422,20 +442,22 @@ pub async fn reload_object_store_setting(db: &windmill_common::DB) -> ObjectStor tracing::error!("S3 cache is not available for pro plan"); return ObjectStoreReload::Never; } - *s3_cache_settings = build_s3_client_from_settings(S3Settings { - bucket: None, - region: None, - access_key: None, - secret_key: None, - endpoint: None, - store_logs: None, - path_style: None, - allow_http: None, - port: None, - }) + *s3_cache_settings = build_object_store_from_settings( + ObjectSettings::S3(S3Settings { + bucket: None, + region: None, + access_key: None, + secret_key: None, + endpoint: None, + store_logs: None, + path_style: None, + allow_http: None, + port: None, + }), + Some(db), + ) .await .ok() - .map(|x| ExpirableObjectStore::from(x)) } else { *s3_cache_settings = None; } @@ -887,19 +909,49 @@ impl ObjectStore for FilesystemStoreIgnoringAttributes { } } +/// What locates a store's objects: endpoint, port, region and bucket (or account and +/// container, or root), never the credentials, which rotate. Two stores with the same +/// location hold the same objects. +pub fn object_store_location(resource: &ObjectStoreResource) -> String { + match resource { + ObjectStoreResource::S3(s) => format!( + "s3:{}:{}:{}:{}", + s.endpoint, + s.port.unwrap_or_default(), + s.region, + s.bucket + ), + ObjectStoreResource::Azure(a) => format!( + "azure:{}:{}:{}", + a.endpoint.as_deref().unwrap_or_default(), + a.account_name, + a.container_name + ), + ObjectStoreResource::Gcs(g) => format!("gcs:{}", g.bucket), + ObjectStoreResource::Filesystem(f) => format!("fs:{}", f.root_path), + } +} + #[cfg(feature = "parquet")] pub async fn build_object_store_from_settings( settings: ObjectSettings, init_private_key: Option<&windmill_common::DB>, ) -> error::Result { + let located = + |store: Arc, resource: ObjectStoreResource| ExpirableObjectStore { + store, + refresh: None, + location: Some(object_store_location(&resource)), + }; match settings { - ObjectSettings::S3(s3_settings) => build_s3_client_from_settings(s3_settings) - .await - .map(|x| ExpirableObjectStore::from(x)), - ObjectSettings::Azure(azure_settings) => { - let azure_blob_resource = azure_settings; - build_azure_blob_client(&azure_blob_resource).map(|x| ExpirableObjectStore::from(x)) + ObjectSettings::S3(s3_settings) => { + let s3_resource = s3_resource_from_settings(s3_settings); + build_s3_client(&s3_resource) + .await + .map(|x| located(x, ObjectStoreResource::S3(s3_resource))) } + ObjectSettings::Azure(azure_settings) => build_azure_blob_client(&azure_settings) + .map(|x| located(x, ObjectStoreResource::Azure(azure_settings))), ObjectSettings::AwsOidc(ref s3_aws_oidc_settings) => { let token_generator = crate::job_s3_helpers_oss::TokenGenerator::AsServerInstance(); let res = crate::job_s3_helpers_oss::generate_s3_aws_oidc_resource( @@ -914,17 +966,14 @@ pub async fn build_object_store_from_settings( .map(|x| ExpirableObjectStore { store: x, refresh: Some(ObjectStoreRefresh::new(settings.clone(), res.expiration())), + location: Some(object_store_location(&res)), }) } - ObjectSettings::Gcs(gcs_settings) => { - let gcs_resource = gcs_settings; - build_gcs_client(&gcs_resource) - .await - .map(|x| ExpirableObjectStore::from(x)) - } - ObjectSettings::Filesystem(fs) => { - build_filesystem_client(&fs.root_path).map(|x| ExpirableObjectStore::from(x)) - } + ObjectSettings::Gcs(gcs_settings) => build_gcs_client(&gcs_settings) + .await + .map(|x| located(x, ObjectStoreResource::Gcs(gcs_settings))), + ObjectSettings::Filesystem(fs) => build_filesystem_client(&fs.root_path) + .map(|x| located(x, ObjectStoreResource::Filesystem(fs))), } } @@ -937,14 +986,14 @@ fn none_if_empty(s: Option) -> Option { } } +/// The S3 resource instance settings resolve to, the environment filling in what they +/// leave out. #[cfg(feature = "parquet")] -pub async fn build_s3_client_from_settings( - settings: S3Settings, -) -> error::Result> { +fn s3_resource_from_settings(settings: S3Settings) -> S3Resource { let region = none_if_empty(settings.region) .unwrap_or_else(|| std::env::var("AWS_REGION").unwrap_or_else(|_| "us-east-1".to_string())); - let s3_resource = S3Resource { + S3Resource { endpoint: none_if_empty(settings.endpoint).unwrap_or_else(|| { std::env::var("S3_ENDPOINT").unwrap_or_else(|_| format!("s3.{region}.amazonaws.com")) }), @@ -959,9 +1008,7 @@ pub async fn build_s3_client_from_settings( port: settings.port, token: None, expiration: None, - }; - - build_s3_client(&s3_resource).await + } } // Resolving the default chain goes over the network (ECS/IMDS) on instances relying on an @@ -2624,6 +2671,36 @@ mod tests { reload_cache_object_store_override(&db, None).await; } + /// A store built from settings is located by where its objects are, not by how the + /// client describes itself: an S3 client prints only its bucket, so the same bucket name + /// on another endpoint would otherwise pass for the same store. + #[cfg(feature = "parquet")] + #[tokio::test] + async fn test_settings_store_location_tells_endpoints_apart() { + let s3 = |endpoint: &str| { + ObjectSettings::S3(S3Settings { + bucket: Some("windmill".to_string()), + region: Some("us-east-1".to_string()), + access_key: Some("key".to_string()), + secret_key: Some("secret".to_string()), + endpoint: Some(endpoint.to_string()), + allow_http: Some(true), + path_style: Some(true), + store_logs: None, + port: None, + }) + }; + let a = build_object_store_from_settings(s3("minio.internal:9000"), None) + .await + .unwrap(); + let b = build_object_store_from_settings(s3("s3.us-east-1.amazonaws.com"), None) + .await + .unwrap(); + assert_eq!(a.store.to_string(), b.store.to_string()); + assert!(a.location.is_some()); + assert_ne!(a.location, b.location); + } + // --- get_logs_from_store test --- #[cfg(feature = "parquet")] diff --git a/docs/ai-session-backups.md b/docs/ai-session-backups.md index 0b4c550456..3e0c747b8a 100644 --- a/docs/ai-session-backups.md +++ b/docs/ai-session-backups.md @@ -129,11 +129,58 @@ against it for the same reason. The feature is on wherever the workspace has primary storage, and off with `ai_config.sessions_storage_disabled` (the `copilot_disabled` pattern: no migration, carried by settings export and the CLI). A build without `parquet` has no routes (404), a workspace without -storage answers `enabled: false`; either turns the backup off for ten minutes, after which the +storage and nothing to stand in for it answers `enabled: false`; either turns the backup off +for ten minutes, after which the page asks again on its own (a flush for whatever is pending, and a restore), and the AI settings page tells the mirror at once when the switch is saved there (the off state is forgotten, the rows that went stale are marked again, a restore runs). +## The instance store standing in + +A workspace without storage of its own keeps its backups in the instance object store +(`object_store_cache_config`, loaded the way every other use of it is, so never with +`DISABLE_S3_STORE`; the plan is checked on every request and Pro never falls back, since a +store loaded before a switch to Pro stays loaded), under the same layout and the same +per-user key, +while the instance setting `ai_sessions_instance_storage_fallback` allows it (on unless set +to false; the instance settings page shows it under Object Storage). A build without +`private` has neither workspace storage nor the quota below, and never falls back. Every +answer says which kind of store it came from (`fallback`), and the instance store is named +(`storage_id`) by what locates its objects, the endpoint, region and bucket its settings +resolve to, in a namespace of its own: moving the instance store to another endpoint under +the same bucket name is a storage switch for the browsers, and a workspace bucket is never +taken for it. The location is kept with the loaded store, so a server whose reload is still +pending names the store it writes to. A route decides between the workspace's storage and +the instance store from the row it reads the generation from, so a push lands in the +instance store only under a generation read while the workspace had no storage. The store a +workspace's backups live in is resolved in one place +(`ai_session_backups::workspace_store`), for the routes and for the rotation's deletion of +older generations alike. + +Configuring a storage for a workspace that had none (`edit_large_file_storage_config`) bumps +the backup generation in the transaction that sets it, so everything the workspace left in +any instance store sits under a generation the routes never read again: a later return to +the instance store, whichever it is by then, starts from a newer one. Every storage settings +change then deletes from the instance store, off the request, the workspace's generations +older than the one it committed, whether the setting is on or off (copies from when it was +on may be there). Nothing live is older, whatever happens next: a deletion that is slow, cut +short, or overtaken by the storage being dropped or pointed at the instance store's own +bucket touches only generations nothing reads. Dropping the storage bumps nothing and is a +switch like any other: the rows go stale, the sessions are pushed whole into the instance +store, and the old bucket keeps its copy. On the browser side a removal owed to an instance +store (the row names it apart, `storageName` in `sessionMirrorPlan.ts`) is retired by any +answer from the workspace's own storage, since that storage being there means the +generation moved past the copy; one owed to a workspace storage still waits for that +storage, whatever the instance store answered. Copies a deletion missed stay in the +operator's bucket unread, as a deleted workspace's copies do. + +On CE the bytes in the instance store count toward the workspace's storage quota under a +storage name of their own (`_ai_sessions_fallback_`, listed by the periodic recount while +the workspace has no storage of its own, and left out when there are none), so a member +cannot fill the operator's bucket past what the workspace may use; on EE, where workspace +storage has no quota either, nothing bounds them but the per-push caps and the instance +setting. + ## Conflicts and deletion Last write wins across devices. The head carries no manifest; `pull` lists the session's prefix @@ -172,7 +219,8 @@ session whose pieces could not be written: recording it would let the next flush half-empty local state over the backup. Every answer names the storage it came from (`storage_id`, a hash of what locates the objects, -endpoint, region and bucket, not the credentials, which rotate) and the backup generation a +endpoint, region and bucket, not the credentials, which rotate; the instance store standing +in for a workspace without one is named apart, see above) and the backup generation a key rotation bumps (`backup_generation`). A sync row records both, and a row naming another storage or generation goes stale and its session is marked again: a workspace pointed at a new bucket, or whose key was rotated, holds nothing, and the server looks nowhere else, so diff --git a/frontend/src/lib/components/InstanceSettings.svelte b/frontend/src/lib/components/InstanceSettings.svelte index f51e459f77..f38c0d3892 100644 --- a/frontend/src/lib/components/InstanceSettings.svelte +++ b/frontend/src/lib/components/InstanceSettings.svelte @@ -134,11 +134,15 @@ } applyFormDefaults(nvalues) - // Apply select/select_python defaults so initialValues matches what InstanceSetting's $effect does + // Apply declared defaults before snapshotting initialValues, so a default shows without + // marking the form dirty: a select's mirrors InstanceSetting's $effect, a boolean's is + // what its toggle shows while the key is unset. for (const category of settingsKeys) { for (const s of settings[category]) { if ( - (s.fieldType === 'select' || s.fieldType === 'select_python') && + (s.fieldType === 'select' || + s.fieldType === 'select_python' || + s.fieldType === 'boolean') && nvalues[s.key] == undefined && s.defaultValue ) { diff --git a/frontend/src/lib/components/instanceSettings.ts b/frontend/src/lib/components/instanceSettings.ts index 0677648978..34079eae51 100644 --- a/frontend/src/lib/components/instanceSettings.ts +++ b/frontend/src/lib/components/instanceSettings.ts @@ -563,6 +563,17 @@ export const settings: Record = { storage: 'setting', ee_only: '' }, + { + label: 'Back AI sessions up to the instance object storage', + description: + "Browsers back their AI sessions up to their workspace's object storage, encrypted with the workspace key. When this is on and instance object storage is configured, a workspace without object storage of its own uses the instance object storage instead, under the same encryption; configuring a storage for the workspace moves its backups there and deletes what it kept in the instance storage. On by default; turn off to keep the AI sessions of such workspaces in the browser only.", + key: 'ai_sessions_instance_storage_fallback', + fieldType: 'boolean', + defaultValue: () => true, + storage: 'setting', + ee_only: '', + hideInQuickSetup: true + }, { label: 'Store audit logs in object storage', description: diff --git a/frontend/src/lib/components/sessions/sessionMirror.svelte.ts b/frontend/src/lib/components/sessions/sessionMirror.svelte.ts index b1ac4bcd06..a18e2febd6 100644 --- a/frontend/src/lib/components/sessions/sessionMirror.svelte.ts +++ b/frontend/src/lib/components/sessions/sessionMirror.svelte.ts @@ -55,10 +55,12 @@ import { import { artifactsFingerprint, headSig, + isFallbackStorage, jsonBytes, operationsOf, planSessionPush, splitEntry, + storageName, type ChatSnapshot, type MirrorSyncState, type PlannedPush, @@ -682,9 +684,10 @@ async function pushWorkspace( return 'transient' } if (!res.enabled) return 'off' - out.storageId = res.storage_id + const storageId = storageName(res.storage_id, res.fallback) + out.storageId = storageId out.generation = res.backup_generation - const answered = `${res.storage_id}:${res.backup_generation}` + const answered = `${storageId}:${res.backup_generation}` const errors = new Set() for (const r of res.results) { if (r.error) { @@ -705,7 +708,7 @@ async function pushWorkspace( // settle, the session goes again whole. if (a.answered !== undefined && a.answered !== answered) failed.add(entry.id) a.answered = answered - a.storageId = res.storage_id + a.storageId = storageId a.generation = res.backup_generation if (errors.has(entry.id)) failed.add(entry.id) } @@ -718,12 +721,20 @@ async function pushWorkspace( [mark.storageId, ...(mark.alsoIn ?? [])].filter((s): s is string => s !== undefined) ) // Answered from a storage holding no copy: the copies are still where they - // were, and the mark waits for those storages to answer. - if (holding.size === 0 || res.storage_id === undefined) out.removedDone.push(mark) - else if (holding.has(res.storage_id)) { - holding.delete(res.storage_id) + // were, and the mark waits for those storages to answer. The workspace's own + // storage answering retires every instance store's share too: configuring it + // moved the generation past all the workspace left in any instance store. + if (holding.size === 0 || storageId === undefined) out.removedDone.push(mark) + else { + const before = holding.size + holding.delete(storageId) + if (!res.fallback) { + for (const name of [...holding]) if (isFallbackStorage(name)) holding.delete(name) + } if (holding.size === 0) out.removedDone.push(mark) - else out.removedFrom.push({ id, key: mark.key, remaining: [...holding] }) + else if (holding.size < before) { + out.removedFrom.push({ id, key: mark.key, remaining: [...holding] }) + } } } } @@ -1204,7 +1215,7 @@ async function listWorkspace(ws: string, email: string): Promise { expect((await __syncRowsForTesting(EMAIL)).some((r) => r.id === 'sr3')).toBe(false) }) + it("retires a removal owed to any instance store once the workspace's own storage answered", async () => { + const s: Session = { id: 'fb1', name: 'session-1', createdAt: 1, workspace_id: 'ws' } + sessionState.sessions = [s] + await putSession(s) + pushMock.mockResolvedValueOnce({ + enabled: true, + storage_id: 'I1', + fallback: true, + results: [{ id: 'fb1' }] + }) + await __flushForTesting() + // The operator moved the instance store: the session goes whole to the new one, and + // the row remembers the copy the old one keeps. + await putSession({ ...s, summary: 'changed' }) + pushMock.mockResolvedValue({ + enabled: true, + storage_id: 'I2', + fallback: true, + results: [{ id: 'fb1' }] + }) + await __flushForTesting() + await __flushForTesting() + const row = (await __syncRowsForTesting(EMAIL)).find((r) => r.id === 'fb1') + expect(row?.storageId).toBe('instance:I2') + expect(row?.alsoIn).toEqual(['instance:I1']) + + // The workspace got a storage of its own meanwhile, which moved the generation past + // everything it left in either instance store: that storage's answer settles the + // removal. + deleteSession('fb1') + await flush() + pushMock.mockResolvedValue({ enabled: true, storage_id: 'A', results: [{ id: 'fb1' }] }) + await __flushForTesting() + expect(pushMock.mock.lastCall?.[0].requestBody.removed).toEqual(['fb1']) + expect(removalKeys()).toEqual([]) + expect((await __syncRowsForTesting(EMAIL)).some((r) => r.id === 'fb1')).toBe(false) + }) + + it('waits for the storage a workspace dropped, whatever the instance store answered', async () => { + const s: Session = { id: 'fb2', name: 'session-1', createdAt: 1, workspace_id: 'ws' } + sessionState.sessions = [s] + await putSession(s) + pushMock.mockResolvedValueOnce({ enabled: true, storage_id: 'A', results: [{ id: 'fb2' }] }) + await __flushForTesting() + // The workspace dropped its storage: the session goes whole to the instance store, + // and the row remembers the copy A keeps. + await putSession({ ...s, summary: 'changed' }) + pushMock.mockResolvedValue({ + enabled: true, + storage_id: 'I', + fallback: true, + results: [{ id: 'fb2' }] + }) + await __flushForTesting() + await __flushForTesting() + const row = (await __syncRowsForTesting(EMAIL)).find((r) => r.id === 'fb2') + expect(row?.storageId).toBe('instance:I') + expect(row?.alsoIn).toEqual(['A']) + + // Deleted while on the instance store: its copy goes, and the mark waits for A. + deleteSession('fb2') + await flush() + await __flushForTesting() + expect(removalKeys()).toEqual(['r::fb2::ws']) + expect((await __syncRowsForTesting(EMAIL)).find((r) => r.id === 'fb2')?.storageId).toBe('A') + pushMock.mockResolvedValue({ enabled: true, storage_id: 'A', results: [{ id: 'fb2' }] }) + await __flushForTesting() + expect(pushMock.mock.lastCall?.[0].requestBody.removed).toEqual(['fb2']) + expect(removalKeys()).toEqual([]) + }) + it("removes a moved session's old copy from the storage that held it, whatever its old workspace is on now", async () => { const s: Session = { id: 'mv2', name: 'session-1', createdAt: 1, workspace_id: 'ws' } sessionState.sessions = [s] diff --git a/frontend/src/lib/components/sessions/sessionMirrorPlan.ts b/frontend/src/lib/components/sessions/sessionMirrorPlan.ts index c61bcf2763..a664e61f5e 100644 --- a/frontend/src/lib/components/sessions/sessionMirrorPlan.ts +++ b/frontend/src/lib/components/sessions/sessionMirrorPlan.ts @@ -31,9 +31,9 @@ export interface MirrorSyncState { /** The user deleted the session and its removal mark could not be written to * localStorage (full): the row itself carries the removal, until it lands. */ removed?: boolean - /** The storage the push landed in, as the server names it, and the backup generation - * (bumped by a workspace key rotation) it landed under. A row recorded against another - * storage or generation describes objects the server no longer looks at. */ + /** The storage the push landed in (`storageName`), and the backup generation (bumped by + * a workspace key rotation) it landed under. A row recorded against another storage or + * generation describes objects the server no longer looks at. */ storageId?: string generation?: number /** Other storages this workspace was on that still hold a copy of the backup (a switch @@ -48,6 +48,34 @@ export interface MirrorSyncState { staging?: { chats: string[]; images: string[]; items: string[]; versions: string[] } } +const FALLBACK_STORAGE_PREFIX = 'instance:' + +/** + * How a storage the server answered from is named in the sync rows and the removal marks: + * by the id the server gives it, the instance object store standing in for a workspace + * without storage of its own (`fallback` on the answer) told apart from a workspace's own. + * A removal owed to an instance store is retired by any answer from the workspace's own + * storage (configuring one moves the backup generation past everything the workspace left + * in any instance store, so none of it is read again), where one owed to a workspace + * storage waits for that storage. + */ +export function storageName(id: string, fallback: boolean | undefined): string +export function storageName( + id: string | undefined, + fallback: boolean | undefined +): string | undefined +export function storageName( + id: string | undefined, + fallback: boolean | undefined +): string | undefined { + if (id === undefined) return undefined + return fallback ? FALLBACK_STORAGE_PREFIX + id : id +} + +export function isFallbackStorage(name: string): boolean { + return name.startsWith(FALLBACK_STORAGE_PREFIX) +} + /** * The part of a session record the backup keeps. Left out on purpose: `name` (a * per-browser counter), the unsent-draft fields (`pending_*`, `draftPrompt`, diff --git a/frontend/src/lib/components/workspaceSettings/AISettings.svelte b/frontend/src/lib/components/workspaceSettings/AISettings.svelte index 7e8b9cc76e..5bc7974af2 100644 --- a/frontend/src/lib/components/workspaceSettings/AISettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/AISettings.svelte @@ -661,7 +661,7 @@ Date: Tue, 15 Sep 2026 23:21:53 +0200 Subject: [PATCH 31/44] fix: stop reading an array job result as wm_failure or http response (#11154) * fix: only read wm_failure and wm_labels from an object job result * fix: serve an array sync result as json, not a composite response --- backend/windmill-api-jobs/src/execution.rs | 17 ++++-- backend/windmill-queue/src/jobs.rs | 52 +++++++++++++++++-- .../windmill-worker/src/result_processor.rs | 10 ++-- 3 files changed, 66 insertions(+), 13 deletions(-) diff --git a/backend/windmill-api-jobs/src/execution.rs b/backend/windmill-api-jobs/src/execution.rs index e897bf5eaf..310e44ae1f 100644 --- a/backend/windmill-api-jobs/src/execution.rs +++ b/backend/windmill-api-jobs/src/execution.rs @@ -38,8 +38,8 @@ use windmill_common::{ FlowVersionInfo, DB, }; use windmill_queue::{ - cancel_job, get_result_and_success_by_id_from_flow, push, PushArgs, PushArgsOwned, - PushIsolationLevel, + cancel_job, get_result_and_success_by_id_from_flow, parse_result_object, push, PushArgs, + PushArgsOwned, PushIsolationLevel, }; use crate::types::RunJobQuery; @@ -374,9 +374,9 @@ pub async fn run_wait_result_internal( } pub fn result_to_response(result: Box, success: bool) -> error::Result { - let composite_result = serde_json::from_str::(result.get()); + let composite_result = parse_result_object::(result.get()); match composite_result { - Ok(WindmillCompositeResult { + Some(WindmillCompositeResult { windmill_status_code, windmill_content_type, windmill_headers, @@ -1192,4 +1192,13 @@ mod result_to_response_tests { assert!(res.is_err(), "hop-by-hop header must be rejected: {name}"); } } + + #[tokio::test] + async fn array_result_is_not_a_composite_response() { + let json = r#"[201,"text/html",null,null,"

    hi

    "]"#; + let resp = result_to_response(raw(json), true).expect("response"); + + assert_eq!(resp.status(), StatusCode::OK); + assert_eq!(body_bytes(resp).await, json.as_bytes()); + } } diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index c4058dba6f..ab91d54287 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -654,6 +654,16 @@ pub struct ResultMetadata { pub wm_failure: Option, } +/// Parses a marker struct out of a job result, which only an object can carry. +/// A derived `Deserialize` also accepts an array, filling fields by position, so +/// without the check a result like `[[], "boom"]` reads as `wm_failure: "boom"`. +pub fn parse_result_object(result: &str) -> Option { + if !result.trim_start().starts_with('{') { + return None; + } + serde_json::from_str(result).ok() +} + /// Sentinel `error.name` we inject into a result when retagging a successful /// run as a failure due to `wm_failure`. Used downstream to detect that /// the result is already in the standard `{ error: { name, message }, ... }` @@ -674,8 +684,7 @@ pub fn is_pre_shaped_wm_failure_result(result: &str) -> bool { struct NameOnly { name: String, } - serde_json::from_str::(result) - .ok() + parse_result_object::(result) .and_then(|m| m.error) .map(|e| e.name == MANUAL_FAILURE_ERROR_NAME) .unwrap_or(false) @@ -721,7 +730,7 @@ impl ValidableJson for Box { } fn result_metadata(&self) -> ResultMetadata { - serde_json::from_str::(self.get()).unwrap_or_default() + parse_result_object::(self.get()).unwrap_or_default() } fn size(&self) -> usize { @@ -774,6 +783,10 @@ impl ValidableJson for serde_json::Value { } fn result_metadata(&self) -> ResultMetadata { + // An array would decode positionally, see `parse_result_object`. + if !self.is_object() { + return ResultMetadata::default(); + } serde_json::from_value::(self.clone()).unwrap_or_default() } @@ -7876,3 +7889,36 @@ mod git_sync_concurrency_key_tests { assert!(a.len() <= 255 && b.len() <= 255); } } + +#[cfg(test)] +mod result_metadata_tests { + use super::{ResultMetadata, ValidableJson}; + use serde_json::value::RawValue; + + fn from_raw(json: &str) -> ResultMetadata { + RawValue::from_string(json.to_string()) + .unwrap() + .result_metadata() + } + + fn from_value(json: &str) -> ResultMetadata { + serde_json::from_str::(json) + .unwrap() + .result_metadata() + } + + #[test] + fn array_result_carries_no_markers() { + for json in [r#"[["label"], "boom"]"#, r#"[null, "boom"]"#] { + for meta in [from_raw(json), from_value(json)] { + assert!( + meta.wm_labels.is_none() && meta.wm_failure.is_none(), + "{json}" + ); + } + } + let meta = from_raw(r#"{"wm_labels": ["label"], "wm_failure": "boom"}"#); + assert_eq!(meta.wm_labels, Some(vec!["label".to_string()])); + assert_eq!(meta.wm_failure.as_deref(), Some("boom")); + } +} diff --git a/backend/windmill-worker/src/result_processor.rs b/backend/windmill-worker/src/result_processor.rs index a269e72f46..1529d474fa 100644 --- a/backend/windmill-worker/src/result_processor.rs +++ b/backend/windmill-worker/src/result_processor.rs @@ -32,8 +32,8 @@ use windmill_common::bench::{BenchmarkInfo, BenchmarkIter}; use windmill_queue::{ append_logs, asset_dispatch, get_mini_completed_job, is_pre_shaped_wm_failure_result, - CanceledBy, FlowRunners, JobCompleted, MiniCompletedJob, MiniPulledJob, ValidableJson, - WrappedError, INIT_SCRIPT_TAG, MANUAL_FAILURE_ERROR_NAME, + parse_result_object, CanceledBy, FlowRunners, JobCompleted, MiniCompletedJob, MiniPulledJob, + ValidableJson, WrappedError, INIT_SCRIPT_TAG, MANUAL_FAILURE_ERROR_NAME, }; use serde_json::{json, value::RawValue, Value}; @@ -72,13 +72,11 @@ struct NestedErrorMessage { /// named `name`/`message`), and we want OTel to record the ManualFailure /// rather than the user's sibling fields. fn extract_error_message(raw: &str) -> Option { - let nested = serde_json::from_str::(raw) - .ok() - .map(|n| n.error); + let nested = parse_result_object::(raw).map(|n| n.error); if matches!(&nested, Some(em) if em.name == MANUAL_FAILURE_ERROR_NAME) { return nested; } - if let Ok(em) = serde_json::from_str::(raw) { + if let Some(em) = parse_result_object::(raw) { return Some(em); } nested From d0d5b295b9942621d3bbf705f0c9e3b8e8637d68 Mon Sep 17 00:00:00 2001 From: AlexRV12 <71396855+AlexRV12@users.noreply.github.com> Date: Wed, 16 Sep 2026 00:26:16 +0200 Subject: [PATCH 32/44] docs: terminate datatable write examples with .execute() (#11148) Co-authored-by: Claude Opus 5 (1M context) --- cli/src/guidance/skills.gen.ts | 23 ++++++++++--------- system_prompts/auto-generated/prompts.ts | 23 ++++++++++--------- .../auto-generated/skills/raw-app/SKILL.md | 23 ++++++++++--------- system_prompts/base/raw-app.md | 23 ++++++++++--------- 4 files changed, 48 insertions(+), 44 deletions(-) diff --git a/cli/src/guidance/skills.gen.ts b/cli/src/guidance/skills.gen.ts index acedfa982b..60fdd07124 100644 --- a/cli/src/guidance/skills.gen.ts +++ b/cli/src/guidance/skills.gen.ts @@ -6012,8 +6012,8 @@ export async function main(user_id: string) { const users = await sql\`SELECT * FROM users WHERE active = \${true}\`.fetch(); // Insert/Update - await sql\`INSERT INTO users (name, email) VALUES (\${name}, \${email})\`; - await sql\`UPDATE users SET name = \${newName} WHERE id = \${user_id}\`; + await sql\`INSERT INTO users (name, email) VALUES (\${name}, \${email})\`.execute(); + await sql\`UPDATE users SET name = \${newName} WHERE id = \${user_id}\`.execute(); return user; } @@ -6032,8 +6032,8 @@ def main(user_id: str): users = db.query('SELECT * FROM users WHERE active = $1', True).fetch() # Insert/Update - db.query('INSERT INTO users (name, email) VALUES ($1, $2)', name, email) - db.query('UPDATE users SET name = $1 WHERE id = $2', new_name, user_id) + db.query('INSERT INTO users (name, email) VALUES ($1, $2)', name, email).execute() + db.query('UPDATE users SET name = $1 WHERE id = $2', new_name, user_id).execute() return user \`\`\` @@ -6042,13 +6042,14 @@ def main(user_id: str): 1. **Check existing tables** before creating new ones — reuse beats schema growth. 2. **Use parameterized queries** — never concatenate user input into SQL. -3. **Keep runnables focused** — one function per runnable; small surface area. -4. **Use descriptive keys** — \`get_user\`, not \`a\`. -5. **Always whitelist tables** — adding a runnable that queries a new table requires the table to be in \`data.tables\` first. -6. **Mark sensitive UI with \`data-wm-no-record\`** — it is what keeps that data out of a recorded demo; passwords are handled for you. -7. **Reach for \`backendAsync\` + \`waitJob\`** for long work — never a hand-written job-polling runnable. -8. **Deploy what a path runnable points at** — a path runnable aimed at a draft fails at runtime; tell the user what needs deploying. -9. **Use \`windmill-chat\` for a chat over a chat-mode flow** — never a runnable that runs the flow and polls its stream. +3. **Terminate every datatable statement** — the tagged template and \`db.query(...)\` only build a statement. It runs when you call \`fetch\` / \`fetchOne\` / \`fetchOneScalar\` / \`execute\` (\`fetch\` / \`fetch_one\` / \`fetch_one_scalar\` / \`execute\` in Python). An INSERT or UPDATE without one writes nothing and raises nothing. Awaiting the statement itself is a no-op — it is not a promise. +4. **Keep runnables focused** — one function per runnable; small surface area. +5. **Use descriptive keys** — \`get_user\`, not \`a\`. +6. **Always whitelist tables** — adding a runnable that queries a new table requires the table to be in \`data.tables\` first. +7. **Mark sensitive UI with \`data-wm-no-record\`** — it is what keeps that data out of a recorded demo; passwords are handled for you. +8. **Reach for \`backendAsync\` + \`waitJob\`** for long work — never a hand-written job-polling runnable. +9. **Deploy what a path runnable points at** — a path runnable aimed at a draft fails at runtime; tell the user what needs deploying. +10. **Use \`windmill-chat\` for a chat over a chat-mode flow** — never a runnable that runs the flow and polls its stream. `, "triggers": `--- name: triggers diff --git a/system_prompts/auto-generated/prompts.ts b/system_prompts/auto-generated/prompts.ts index 4b4552dca4..2a20e8f779 100644 --- a/system_prompts/auto-generated/prompts.ts +++ b/system_prompts/auto-generated/prompts.ts @@ -920,8 +920,8 @@ export async function main(user_id: string) { const users = await sql\`SELECT * FROM users WHERE active = \${true}\`.fetch(); // Insert/Update - await sql\`INSERT INTO users (name, email) VALUES (\${name}, \${email})\`; - await sql\`UPDATE users SET name = \${newName} WHERE id = \${user_id}\`; + await sql\`INSERT INTO users (name, email) VALUES (\${name}, \${email})\`.execute(); + await sql\`UPDATE users SET name = \${newName} WHERE id = \${user_id}\`.execute(); return user; } @@ -940,8 +940,8 @@ def main(user_id: str): users = db.query('SELECT * FROM users WHERE active = $1', True).fetch() # Insert/Update - db.query('INSERT INTO users (name, email) VALUES ($1, $2)', name, email) - db.query('UPDATE users SET name = $1 WHERE id = $2', new_name, user_id) + db.query('INSERT INTO users (name, email) VALUES ($1, $2)', name, email).execute() + db.query('UPDATE users SET name = $1 WHERE id = $2', new_name, user_id).execute() return user \`\`\` @@ -950,13 +950,14 @@ def main(user_id: str): 1. **Check existing tables** before creating new ones — reuse beats schema growth. 2. **Use parameterized queries** — never concatenate user input into SQL. -3. **Keep runnables focused** — one function per runnable; small surface area. -4. **Use descriptive keys** — \`get_user\`, not \`a\`. -5. **Always whitelist tables** — adding a runnable that queries a new table requires the table to be in \`data.tables\` first. -6. **Mark sensitive UI with \`data-wm-no-record\`** — it is what keeps that data out of a recorded demo; passwords are handled for you. -7. **Reach for \`backendAsync\` + \`waitJob\`** for long work — never a hand-written job-polling runnable. -8. **Deploy what a path runnable points at** — a path runnable aimed at a draft fails at runtime; tell the user what needs deploying. -9. **Use \`windmill-chat\` for a chat over a chat-mode flow** — never a runnable that runs the flow and polls its stream. +3. **Terminate every datatable statement** — the tagged template and \`db.query(...)\` only build a statement. It runs when you call \`fetch\` / \`fetchOne\` / \`fetchOneScalar\` / \`execute\` (\`fetch\` / \`fetch_one\` / \`fetch_one_scalar\` / \`execute\` in Python). An INSERT or UPDATE without one writes nothing and raises nothing. Awaiting the statement itself is a no-op — it is not a promise. +4. **Keep runnables focused** — one function per runnable; small surface area. +5. **Use descriptive keys** — \`get_user\`, not \`a\`. +6. **Always whitelist tables** — adding a runnable that queries a new table requires the table to be in \`data.tables\` first. +7. **Mark sensitive UI with \`data-wm-no-record\`** — it is what keeps that data out of a recorded demo; passwords are handled for you. +8. **Reach for \`backendAsync\` + \`waitJob\`** for long work — never a hand-written job-polling runnable. +9. **Deploy what a path runnable points at** — a path runnable aimed at a draft fails at runtime; tell the user what needs deploying. +10. **Use \`windmill-chat\` for a chat over a chat-mode flow** — never a runnable that runs the flow and polls its stream. `; export const PIPELINE_BASE = `# Data pipeline authoring diff --git a/system_prompts/auto-generated/skills/raw-app/SKILL.md b/system_prompts/auto-generated/skills/raw-app/SKILL.md index da4a9729be..ed79172b9d 100644 --- a/system_prompts/auto-generated/skills/raw-app/SKILL.md +++ b/system_prompts/auto-generated/skills/raw-app/SKILL.md @@ -435,8 +435,8 @@ export async function main(user_id: string) { const users = await sql`SELECT * FROM users WHERE active = ${true}`.fetch(); // Insert/Update - await sql`INSERT INTO users (name, email) VALUES (${name}, ${email})`; - await sql`UPDATE users SET name = ${newName} WHERE id = ${user_id}`; + await sql`INSERT INTO users (name, email) VALUES (${name}, ${email})`.execute(); + await sql`UPDATE users SET name = ${newName} WHERE id = ${user_id}`.execute(); return user; } @@ -455,8 +455,8 @@ def main(user_id: str): users = db.query('SELECT * FROM users WHERE active = $1', True).fetch() # Insert/Update - db.query('INSERT INTO users (name, email) VALUES ($1, $2)', name, email) - db.query('UPDATE users SET name = $1 WHERE id = $2', new_name, user_id) + db.query('INSERT INTO users (name, email) VALUES ($1, $2)', name, email).execute() + db.query('UPDATE users SET name = $1 WHERE id = $2', new_name, user_id).execute() return user ``` @@ -465,10 +465,11 @@ def main(user_id: str): 1. **Check existing tables** before creating new ones — reuse beats schema growth. 2. **Use parameterized queries** — never concatenate user input into SQL. -3. **Keep runnables focused** — one function per runnable; small surface area. -4. **Use descriptive keys** — `get_user`, not `a`. -5. **Always whitelist tables** — adding a runnable that queries a new table requires the table to be in `data.tables` first. -6. **Mark sensitive UI with `data-wm-no-record`** — it is what keeps that data out of a recorded demo; passwords are handled for you. -7. **Reach for `backendAsync` + `waitJob`** for long work — never a hand-written job-polling runnable. -8. **Deploy what a path runnable points at** — a path runnable aimed at a draft fails at runtime; tell the user what needs deploying. -9. **Use `windmill-chat` for a chat over a chat-mode flow** — never a runnable that runs the flow and polls its stream. +3. **Terminate every datatable statement** — the tagged template and `db.query(...)` only build a statement. It runs when you call `fetch` / `fetchOne` / `fetchOneScalar` / `execute` (`fetch` / `fetch_one` / `fetch_one_scalar` / `execute` in Python). An INSERT or UPDATE without one writes nothing and raises nothing. Awaiting the statement itself is a no-op — it is not a promise. +4. **Keep runnables focused** — one function per runnable; small surface area. +5. **Use descriptive keys** — `get_user`, not `a`. +6. **Always whitelist tables** — adding a runnable that queries a new table requires the table to be in `data.tables` first. +7. **Mark sensitive UI with `data-wm-no-record`** — it is what keeps that data out of a recorded demo; passwords are handled for you. +8. **Reach for `backendAsync` + `waitJob`** for long work — never a hand-written job-polling runnable. +9. **Deploy what a path runnable points at** — a path runnable aimed at a draft fails at runtime; tell the user what needs deploying. +10. **Use `windmill-chat` for a chat over a chat-mode flow** — never a runnable that runs the flow and polls its stream. diff --git a/system_prompts/base/raw-app.md b/system_prompts/base/raw-app.md index 3849a465a0..f7eaef944d 100644 --- a/system_prompts/base/raw-app.md +++ b/system_prompts/base/raw-app.md @@ -200,8 +200,8 @@ export async function main(user_id: string) { const users = await sql`SELECT * FROM users WHERE active = ${true}`.fetch(); // Insert/Update - await sql`INSERT INTO users (name, email) VALUES (${name}, ${email})`; - await sql`UPDATE users SET name = ${newName} WHERE id = ${user_id}`; + await sql`INSERT INTO users (name, email) VALUES (${name}, ${email})`.execute(); + await sql`UPDATE users SET name = ${newName} WHERE id = ${user_id}`.execute(); return user; } @@ -220,8 +220,8 @@ def main(user_id: str): users = db.query('SELECT * FROM users WHERE active = $1', True).fetch() # Insert/Update - db.query('INSERT INTO users (name, email) VALUES ($1, $2)', name, email) - db.query('UPDATE users SET name = $1 WHERE id = $2', new_name, user_id) + db.query('INSERT INTO users (name, email) VALUES ($1, $2)', name, email).execute() + db.query('UPDATE users SET name = $1 WHERE id = $2', new_name, user_id).execute() return user ``` @@ -230,10 +230,11 @@ def main(user_id: str): 1. **Check existing tables** before creating new ones — reuse beats schema growth. 2. **Use parameterized queries** — never concatenate user input into SQL. -3. **Keep runnables focused** — one function per runnable; small surface area. -4. **Use descriptive keys** — `get_user`, not `a`. -5. **Always whitelist tables** — adding a runnable that queries a new table requires the table to be in `data.tables` first. -6. **Mark sensitive UI with `data-wm-no-record`** — it is what keeps that data out of a recorded demo; passwords are handled for you. -7. **Reach for `backendAsync` + `waitJob`** for long work — never a hand-written job-polling runnable. -8. **Deploy what a path runnable points at** — a path runnable aimed at a draft fails at runtime; tell the user what needs deploying. -9. **Use `windmill-chat` for a chat over a chat-mode flow** — never a runnable that runs the flow and polls its stream. +3. **Terminate every datatable statement** — the tagged template and `db.query(...)` only build a statement. It runs when you call `fetch` / `fetchOne` / `fetchOneScalar` / `execute` (`fetch` / `fetch_one` / `fetch_one_scalar` / `execute` in Python). An INSERT or UPDATE without one writes nothing and raises nothing. Awaiting the statement itself is a no-op — it is not a promise. +4. **Keep runnables focused** — one function per runnable; small surface area. +5. **Use descriptive keys** — `get_user`, not `a`. +6. **Always whitelist tables** — adding a runnable that queries a new table requires the table to be in `data.tables` first. +7. **Mark sensitive UI with `data-wm-no-record`** — it is what keeps that data out of a recorded demo; passwords are handled for you. +8. **Reach for `backendAsync` + `waitJob`** for long work — never a hand-written job-polling runnable. +9. **Deploy what a path runnable points at** — a path runnable aimed at a draft fails at runtime; tell the user what needs deploying. +10. **Use `windmill-chat` for a chat over a chat-mode flow** — never a runnable that runs the flow and polls its stream. From a9a9335a34a13ffd8cd2699adc92087b679548ca Mon Sep 17 00:00:00 2001 From: Guilhem Date: Wed, 16 Sep 2026 00:26:56 +0200 Subject: [PATCH 33/44] fix: keep the instance users table's actions and header in view (#11145) * fix: keep the instance users table's actions and header in view Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_015pMjimm9tUMXtD8rkWB62w * fix: lock only the User option for group-granted roles and keep pinned cells opaque Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_015pMjimm9tUMXtD8rkWB62w * fix: close the instance settings drawer from the manage-in-workspace menu item Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_015pMjimm9tUMXtD8rkWB62w --------- Co-authored-by: Claude Fable 5.1 Co-authored-by: Ruben Fiszel --- .../components/ChangeInstanceUsername.svelte | 20 +- .../lib/components/SuperadminSettings.svelte | 2 +- .../components/SuperadminSettingsInner.svelte | 326 ++++++++++-------- .../toggleButton-v2/ToggleButton.svelte | 14 +- .../settings/ForkMemberSettings.svelte | 4 +- .../settings/WorkspaceUserSettings.svelte | 8 +- frontend/src/lib/components/table/Head.svelte | 11 +- 7 files changed, 232 insertions(+), 153 deletions(-) diff --git a/frontend/src/lib/components/ChangeInstanceUsername.svelte b/frontend/src/lib/components/ChangeInstanceUsername.svelte index 1fcd726cea..fce0f4e8f9 100644 --- a/frontend/src/lib/components/ChangeInstanceUsername.svelte +++ b/frontend/src/lib/components/ChangeInstanceUsername.svelte @@ -3,6 +3,7 @@ import Popover from './meltComponents/Popover.svelte' import { autoPlacement } from '@floating-ui/core' import ChangeInstanceUsernameInner from './ChangeInstanceUsernameInner.svelte' + import { AlertTriangle } from 'lucide-svelte' interface Props { email: string @@ -24,9 +25,22 @@ closeButton > {#snippet trigger()} - + {#if isConflict} + + + {/if} {/snippet} {#snippet content()} - + {#snippet titleExtra()} diff --git a/frontend/src/lib/components/SuperadminSettingsInner.svelte b/frontend/src/lib/components/SuperadminSettingsInner.svelte index 97b15fe443..0fd7991597 100644 --- a/frontend/src/lib/components/SuperadminSettingsInner.svelte +++ b/frontend/src/lib/components/SuperadminSettingsInner.svelte @@ -26,11 +26,10 @@ CheckCircle2, ExternalLink, Pencil, + Settings, UserMinus, UserPlus } from 'lucide-svelte' - import Badge from './common/badge/Badge.svelte' - import Tooltip from './Tooltip.svelte' import DropdownV2 from './DropdownV2.svelte' import Popover from './meltComponents/Popover.svelte' import ConfirmationModal from './common/confirmationModal/ConfirmationModal.svelte' @@ -155,6 +154,11 @@ loadExtJwtPage(1) let tab: string = $state('users') + let usersListShown = $derived( + tab === 'users' && + !yamlMode && + (usersSubTab === 'users' || (usersSubTab === 'ext_jwt' && extJwtTokens.length === 0)) + ) $effect(() => { tab = $instanceSettingsSelectedTab @@ -320,11 +324,14 @@
    -
    + +
    {#if tab === 'ai' && !yamlMode} {:else if tab === 'users' && !yamlMode} -
    +
    {#if !automateUsernameCreation && !isCloudHosted()}

    Automatic username creation

    @@ -373,7 +380,7 @@ - {#if usersSubTab === 'users' || (usersSubTab === 'ext_jwt' && extJwtTokens.length === 0)} + {#if usersListShown} {filteredUsers.length} user{filteredUsers.length !== 1 ? 's' : ''} found

    -
    - 50} - loadMore={50} - on:loadMore={() => { - nbDisplayed += 50 - }} - > + +
    + Email @@ -434,7 +437,7 @@ Kind {/if} Role - + Actions @@ -443,12 +446,22 @@ {#if filteredUsers && users} {#each filteredUsers.slice(0, nbDisplayed) as { email, super_admin, devops, login_type, name, username, operator_only, is_workspace_admin, role_source, disabled, workspace_id }, i (email + '::' + (workspace_id ?? ''))} {@const isServiceAccount = login_type === 'service_account'} + {@const groupRole = + role_source === 'instance_group' && (super_admin || devops)} + + {@const groupRoleTooltip = + 'Role is set by an instance group. Superadmin and Devops can be set here, but demoting to User requires removing the user from the group.'} + {@const serviceAccountTooltip = + 'Service accounts are always users in the instance. Their workspace role is managed in the workspace user settings.'} + - +
    {#if isServiceAccount} @@ -458,14 +471,6 @@ >{email} {/if} - {#if workspace_id} - - {truncate(workspace_id, 20)} - - {/if} {#if disabled} {#if automateUsernameCreation} - + {#if username} {username} {:else} @@ -503,133 +508,157 @@ > {#if activeOnly} - {#if is_workspace_admin} - Admin - {:else if operator_only} - Operator only - {:else} - Developer - {/if} + + {#if is_workspace_admin} + Admin + {:else if operator_only} + Operator only + {:else} + Developer + {/if} + {/if} - {#if isServiceAccount} -
    + +
    + {#key `${super_admin}_${devops}_${role_source}`} + { + if (email == $userStore?.email) { + sendUserToast('You cannot demote yourself', true) + listUsers(activeOnly) + return + } + + let role = e.detail + + if (role === 'super_admin') { + await UserService.globalUserUpdate({ + email, + requestBody: { + is_super_admin: true, + is_devops: false + } + }) + } + if (role === 'devops') { + await UserService.globalUserUpdate({ + email, + requestBody: { + is_super_admin: false, + is_devops: true + } + }) + } + if (role === 'user') { + await UserService.globalUserUpdate({ + email, + requestBody: { + is_super_admin: false, + is_devops: false + } + }) + } + sendUserToast('User updated') + listUsers(activeOnly) + }} + > + {#snippet children({ item })} + + + + {/snippet} + + {/key} + {#if isServiceAccount} {is_workspace_admin ? 'Admin' : operator_only ? 'Operator' : 'Developer'} + in + {#if workspace_id} + closeDrawer?.()} + >{truncate(workspace_id, 20)} + {:else} + its workspace + {/if} - - Service-account role is managed in the workspace user settings. - -
    - {:else} -
    - {#key `${super_admin}_${devops}_${role_source}`} - { - if (email == $userStore?.email) { - sendUserToast('You cannot demote yourself', true) - listUsers(activeOnly) - return - } - - let role = e.detail - - if (role === 'super_admin') { - await UserService.globalUserUpdate({ - email, - requestBody: { - is_super_admin: true, - is_devops: false - } - }) - } - if (role === 'devops') { - await UserService.globalUserUpdate({ - email, - requestBody: { - is_super_admin: false, - is_devops: true - } - }) - } - if (role === 'user') { - await UserService.globalUserUpdate({ - email, - requestBody: { - is_super_admin: false, - is_devops: false - } - }) - } - sendUserToast('User updated') - listUsers(activeOnly) - }} - > - {#snippet children({ item })} - - - - {/snippet} - - {/key} - {#if role_source === 'instance_group' && (super_admin || devops)} - closeDrawer?.()} - > - Set by instance group - - {/if} -
    - {/if} + {:else if groupRole} + closeDrawer?.()} + > + Set by instance group + + {/if} +
    - +
    {#if isServiceAccount} {#if workspace_id} - Manage in workspace + closeDrawer?.(), + href: `${base}/workspace_settings?tab=users&workspace=${workspace_id}` + } + ]} + /> {/if} {:else}
    {/each} + {#if filteredUsers.length > nbDisplayed} + {@const remaining = Math.min(50, filteredUsers.length - nbDisplayed)} + + + + + + + {/if} {/if} diff --git a/frontend/src/lib/components/common/toggleButton-v2/ToggleButton.svelte b/frontend/src/lib/components/common/toggleButton-v2/ToggleButton.svelte index 621e8383d0..2773503df9 100644 --- a/frontend/src/lib/components/common/toggleButton-v2/ToggleButton.svelte +++ b/frontend/src/lib/components/common/toggleButton-v2/ToggleButton.svelte @@ -7,6 +7,9 @@ interface Props { label?: string | undefined + /** Shown instead of `label` below the `xl` breakpoint, for groups that must keep + * their width inside a narrow table cell. The full label stays the accessible name. */ + shortLabel?: string | undefined iconOnly?: boolean tooltip?: string | undefined icon?: any | undefined @@ -30,6 +33,7 @@ let { label = undefined, + shortLabel = undefined, iconOnly = false, tooltip = undefined, icon = undefined, @@ -68,6 +72,7 @@
    - +
    From a48ae656ae59d600311f81ef357d08df5226515a Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 16 Sep 2026 09:57:43 +0200 Subject: [PATCH 35/44] feat: delete a browser's copy of an AI session past its workspace retention (#11156) * feat: delete a browser's copy of an AI session past its workspace retention Co-Authored-By: Claude Opus 5 (1M context) * fix: tell the AI session retention only to a member who can reach the workspace Co-Authored-By: Claude Opus 5 (1M context) * docs: keep the retention sweep's design narrative in the docs, not the code * fix: give the session retention its own route, leaving the status contract alone Co-Authored-By: Claude Opus 5 (1M context) * docs: shorten the retention route comment to its constraints * docs: name the two clocks in the retention setting, and the deploy window --------- Co-authored-by: Claude Opus 5 (1M context) --- ...c8a287bf1ff4b9d4301e3ea0efd8077936aff.json | 30 ++ backend/tests/session_workspace_status.rs | 68 ++++- .../windmill-api-workspaces/src/workspaces.rs | 40 +++ backend/windmill-api/openapi.yaml | 31 ++ docs/ai-session-backups.md | 104 +++++-- .../copilot/chat/HistoryManager.svelte.ts | 17 +- .../chat/artifacts/artifactsDB.test.ts | 2 +- .../copilot/chat/artifacts/artifactsDB.ts | 11 +- .../chat/files/attachedFilesDB.test.ts | 2 +- .../copilot/chat/files/attachedFilesDB.ts | 7 +- .../sessions/sessionMirror.svelte.ts | 12 +- .../components/sessions/sessionMirror.test.ts | 15 +- .../components/sessions/sessionMirrorPlan.ts | 6 +- .../sessions/sessionMirrorSignal.ts | 24 ++ .../sessions/sessionState.svelte.ts | 272 +++++++++++++++++- .../sessions/sessionStateIndexedDb.test.ts | 130 ++++++++- .../workspaceSettings/AISettings.svelte | 2 +- 17 files changed, 718 insertions(+), 55 deletions(-) create mode 100644 backend/.sqlx/query-8eee14066c86b4a4ef921576277c8a287bf1ff4b9d4301e3ea0efd8077936aff.json diff --git a/backend/.sqlx/query-8eee14066c86b4a4ef921576277c8a287bf1ff4b9d4301e3ea0efd8077936aff.json b/backend/.sqlx/query-8eee14066c86b4a4ef921576277c8a287bf1ff4b9d4301e3ea0efd8077936aff.json new file mode 100644 index 0000000000..552a38a0d4 --- /dev/null +++ b/backend/.sqlx/query-8eee14066c86b4a4ef921576277c8a287bf1ff4b9d4301e3ea0efd8077936aff.json @@ -0,0 +1,30 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT workspace_settings.workspace_id AS \"id!\",\n workspace_settings.ai_config->'sessions_retention_days' AS retention\n FROM workspace_settings\n LEFT JOIN usr ON usr.workspace_id = workspace_settings.workspace_id AND usr.email = $2\n WHERE workspace_settings.workspace_id = ANY($1)\n AND ($3 OR (usr.email IS NOT NULL AND NOT usr.disabled))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "retention", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "TextArray", + "Text", + "Bool" + ] + }, + "nullable": [ + false, + null + ] + }, + "hash": "8eee14066c86b4a4ef921576277c8a287bf1ff4b9d4301e3ea0efd8077936aff" +} diff --git a/backend/tests/session_workspace_status.rs b/backend/tests/session_workspace_status.rs index b4fc98da5e..2ed4ac503a 100644 --- a/backend/tests/session_workspace_status.rs +++ b/backend/tests/session_workspace_status.rs @@ -3,18 +3,23 @@ //! extractor actually grants. Membership is not the only path: a superadmin is authed into //! any existing workspace without a `usr` row, and `admins` has no `usr` rows at all, so //! answering from `usr` alone reports live workspaces as unresolvable and the client deletes -//! sessions that still work. +//! sessions that still work. `POST /workspaces/session_workspace_retention`, the AI session +//! retention the same client deletes its own copies by, is a workspace setting and answers to +//! the stricter bar, which is why the two are separate routes and tested together. use serde_json::json; use sqlx::{Pool, Postgres}; use std::collections::HashMap; use windmill_test_utils::*; -async fn status(port: u16, token: &str, ids: &[&str]) -> anyhow::Result> { +async fn post( + port: u16, + route: &str, + token: &str, + ids: &[&str], +) -> anyhow::Result { let resp = reqwest::Client::new() - .post(format!( - "http://localhost:{port}/api/workspaces/session_workspace_status" - )) + .post(format!("http://localhost:{port}/api/workspaces/{route}")) .header("Authorization", format!("Bearer {token}")) .json(&json!({ "workspace_ids": ids })) .send() @@ -23,6 +28,14 @@ async fn status(port: u16, token: &str, ids: &[&str]) -> anyhow::Result anyhow::Result> { + post(port, "session_workspace_status", token, ids).await +} + +async fn retention(port: u16, token: &str, ids: &[&str]) -> anyhow::Result> { + post(port, "session_workspace_retention", token, ids).await +} + #[sqlx::test(fixtures("base", "session_workspace_status"))] async fn test_superadmin_reaches_workspaces_without_a_usr_row( db: Pool, @@ -60,3 +73,48 @@ async fn test_superadmin_reaches_workspaces_without_a_usr_row( Ok(()) } + +/// The retention a browser deletes its own copies by is a workspace setting, so unlike the +/// status it is told only to a caller the authed extractor would let in. +#[sqlx::test(fixtures("base", "session_workspace_status"))] +async fn test_session_retention_is_told_only_to_members_who_can_be_authed( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let ids = ["foreign-workspace", "test-workspace", "no-such-workspace"]; + sqlx::query( + "UPDATE workspace_settings SET ai_config = '{\"sessions_retention_days\": 7}' \ + WHERE workspace_id IN ('test-workspace', 'foreign-workspace')", + ) + .execute(&db) + .await?; + + // test@windmill.dev is a superadmin: authed into every workspace that exists. + let sa = retention(port, "SECRET_TOKEN", &ids).await?; + assert_eq!(sa["test-workspace"], 7); + assert_eq!(sa["foreign-workspace"], 7); + assert!(!sa.contains_key("no-such-workspace")); + + // test2@windmill.dev is a member of test-workspace only. + let usr = retention(port, "SECRET_TOKEN_2", &ids).await?; + assert_eq!(usr["test-workspace"], 7); + assert!(!usr.contains_key("foreign-workspace")); + + // A disabled membership still reconciles its sessions — the status stays `active` — but + // cannot be authed into the workspace, so it is told no setting. + sqlx::query("UPDATE usr SET disabled = true WHERE workspace_id = 'test-workspace'") + .execute(&db) + .await?; + assert_eq!( + status(port, "SECRET_TOKEN_2", &ids).await?["test-workspace"], + "active" + ); + assert!(!retention(port, "SECRET_TOKEN_2", &ids) + .await? + .contains_key("test-workspace")); + + Ok(()) +} diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index c29d99d05b..3330ed697f 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -227,6 +227,10 @@ pub fn global_service() -> Router { .route("/list", get(list_workspaces)) .route("/users", get(user_workspaces)) .route("/session_workspace_status", post(session_workspace_status)) + .route( + "/session_workspace_retention", + post(session_workspace_retention), + ) .route("/create", post(create_workspace)) .route("/create_fork", post(deprecated_create_workspace_fork)) .route("/exists", post(exists_workspace)) @@ -5686,6 +5690,42 @@ async fn session_workspace_status( Ok(Json(statuses)) } +/// The AI session retention a browser deletes its local copies by (docs/ai-session-backups.md). +/// Its own route, not a field on the status above, whose shape an older tab still reads. Unlike +/// a status, it answers only for a workspace this caller can be authed into: a setting is the +/// workspace's to tell, so a disabled membership gets none though its sessions still reconcile. +async fn session_workspace_retention( + Extension(db): Extension, + authed: ApiAuthed, + Json(req): Json, +) -> JsonResult> { + if req.workspace_ids.len() > 1000 { + return Err(Error::BadRequest( + "Too many workspace ids (max 1000)".to_string(), + )); + } + let email = &authed.email; + let is_superadmin = windmill_api_auth::is_super_admin_authed(&db, &authed).await?; + let rows = sqlx::query!( + "SELECT workspace_settings.workspace_id AS \"id!\", + workspace_settings.ai_config->'sessions_retention_days' AS retention + FROM workspace_settings + LEFT JOIN usr ON usr.workspace_id = workspace_settings.workspace_id AND usr.email = $2 + WHERE workspace_settings.workspace_id = ANY($1) + AND ($3 OR (usr.email IS NOT NULL AND NOT usr.disabled))", + &req.workspace_ids[..], + email, + is_superadmin, + ) + .fetch_all(&db) + .await?; + let days = rows + .into_iter() + .filter_map(|r| sessions_retention_days(r.retention.as_ref()).map(|days| (r.id, days))) + .collect(); + Ok(Json(days)) +} + /// The instance critical alert channels belong to the instance operator, who on cloud is /// not the workspace owner and never opted into a tenant's job failures. Fork workspaces run /// throwaway copies of their parent's runnables, so instance-wide operational alerting must diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 9ce1aef272..c88a298646 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1277,6 +1277,37 @@ paths: - archived - deleted + /workspaces/session_workspace_retention: + post: + summary: get the AI session retention of workspaces referenced by client-side sessions + operationId: getSessionWorkspaceRetention + tags: + - workspace + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + workspace_ids: + type: array + items: + type: string + required: + - workspace_ids + responses: + "200": + description: >- + map of workspace id to its `ai_config.sessions_retention_days`; a workspace + without a retention, or one the caller cannot be authenticated into, is absent + content: + application/json: + schema: + type: object + additionalProperties: + type: integer + /w/{workspace}/workspaces/get_as_superadmin: get: summary: get workspace as super admin (require to be super admin) diff --git a/docs/ai-session-backups.md b/docs/ai-session-backups.md index 9af2bd3a84..ffec41e182 100644 --- a/docs/ai-session-backups.md +++ b/docs/ai-session-backups.md @@ -240,32 +240,86 @@ the first push after it or on the next page load, whichever comes first. `ai_config.sessions_retention_days` (per workspace, in the AI settings; unset by default; the `sessions_storage_disabled` pattern: no migration, carried by settings export and the -CLI; 1 to 3650) puts an age on backups, counted from the last push of the session that -completed. It applies to the backup only: a browser keeps its copy whatever the retention, -and a backup swept while a browser still has the session comes back once that browser writes -to it again (its incremental push is refused and goes whole). +CLI; 1 to 3650) puts an age on sessions, counted from their last activity. Each side applies +it with its own clock against its own timestamps, so no clock is compared with another +machine's, and the two do not time the same event: the server counts the last push that +completed, a browser its last local activity, which includes reading new messages and is not +pushed. A backup swept while a browser still reads its copy comes back once that browser +writes to the session again (its incremental push is refused and goes whole): -The server sweeps the object store (`sweep_expired_ai_session_backups`, from the monitor about -every 40 minutes on each server, one pass at a time under a session-level advisory lock). For -every workspace with a retention it takes the store its backups live in, its own storage or -the instance store standing in, decided from the row it reads the generation from as the -routes do, names the users under the generation prefix (`list_with_delimiter`) and lists each -user's `index/` once: one object per session, nothing of what the sessions hold. A session -whose marker is older than the retention is removed under its lock (`lock_session`), once its -markers, listed again there, are still all older: a push that renewed the session between the -walk and the lock keeps it, and one split over parts either holds the lock or has the session -unlisted with its token next to the markers (`index/{sid}/push`), which the sweep leaves -alone while the token is younger than the retention: an older one is a push a browser -abandoned, whose landed parts nothing lists, and it goes the same way. Before deleting -anything the sweep writes a record next to the markers (`index/{sid}/sweep`, not an epoch, so -neither `list` nor `pull` counts it), and `remove_session` deletes it last: a removal cut -short, its markers already gone, is found by the next pass and finished, unless a push listed -the session again first. At most 1000 sessions per workspace and pass; the rest wait for the -next. `list` leaves an expired marker out of its answer meanwhile, so a browser never restores -a session the sweep has not reached. The marker's modification time is the storage's clock and -the cutoff the server's. The sweep reaches only the backups the routes would: a deleted -workspace's stay in its storage, and so do those a workspace keeps in the instance store once -`ai_sessions_instance_storage_fallback` is set to false. +- The server sweeps the object store (`sweep_expired_ai_session_backups`, from the monitor + about every 40 minutes on each server, one pass at a time under a session-level advisory + lock). For every workspace with a retention it takes the store its backups live in, its + own storage or the instance store standing in, decided from the row it reads the + generation from as the routes do, names the users under the generation prefix + (`list_with_delimiter`) and lists each user's `index/` once: one object + per session, nothing of what the sessions hold. A session whose marker is older than the + retention is removed under its lock (`lock_session`), once its markers, listed again + there, are still all older: a push that renewed the session between the walk and the lock + keeps it, and one split over parts either holds the lock or has the session unlisted with + its token next to the markers (`index/{sid}/push`), which the sweep leaves alone while the + token is younger than the retention: an older one is a push a browser abandoned, whose + landed parts nothing lists, and it goes the same way. Before deleting anything the sweep + writes a record next to + the markers (`index/{sid}/sweep`, not an epoch, so neither `list` nor `pull` counts it), + and `remove_session` deletes it last: a removal cut short, its markers already gone, is + found by the next pass and finished, unless a push listed the session again first. At + most 1000 sessions per workspace and pass; the rest wait for the next. `list` leaves an + expired marker out of its answer meanwhile, so a browser never restores a session the + sweep has not reached. The marker's modification time is the storage's clock and the + cutoff the server's. The sweep reaches only the backups the routes would: a deleted + workspace's stay in its storage, and so do those a workspace keeps in the instance store + once `ai_sessions_instance_storage_fallback` is set to false. +- The browser sweeps its own stores when a tab resolves the logged-in user + (`sweepExpiredSessions`, from the one `onUserChange` in `sessionState.svelte.ts`), before + that tab reads a single session. A session whose last activity is older than the retention + by the browser's clock is deleted locally, record, chats, images, attached files and + artifacts. A restored session carries the backup's time as its last activity, the storage's + clock, so it counts from the later of that and the moment it was restored here + (`restoredAt`): a browser clock ahead of the storage's never deletes a session it just + brought back. Archived sessions count like any other, and persisted unsent drafts by their + pending workspace. + + The stores are shared by the user's tabs, and each keeps copies of the sessions in memory, + so every tab holds a shared Web Lock from before it reads them until it stops using them, + and the sweep deletes only while holding that lock exclusively, requested if available: + granted exactly when no tab of the user has the sessions loaded, which is why the sweep + runs where it does and nowhere else. Nothing holds a copy of what it deletes and nothing + writes the stores meanwhile, so it deletes one record at a time and without re-reading. It + also takes the tab lock the flush and the restore take, again only if available, so neither + plans nor stages a session half deleted; like the restore, it does not run where Web Locks + do not exist. With several tabs open nothing is swept, until one of them reloads alone. + + The hold is only as good as the tabs that take it, so a tab still running a build from before + it has the sessions loaded and holds nothing. A tab loaded after that one, across a deploy, + can sweep a session the older tab has in memory, and a write there afterwards brings the + record back without its chats, which the next flush pushes. It needs a tab left open across a + deploy, a session untouched for the whole retention, and the user going back to that session + in the older tab; the next sweep deletes it again. The same window is open to the + workspace-lifecycle delete in `reconcileSessionsLifecycle`, which no lock guards at all. + + What deletes is the retention the server gives as the sweep runs, asked for under both locks + (`POST /workspaces/session_workspace_retention`, its own route rather than a field on the + lifecycle status, whose answer a tab loaded before this version still reads). Never a + remembered one: a retention raised or cleared since would otherwise delete a session that is + now within it, and a persisted unsent draft has no backup to come back from. What the sweep + keeps in localStorage decides only whether to ask again — it asks when it has asked nothing + yet, when the answer it has is a day old, or when that answer marks a session expired — so + an ordinary load costs no request at all. An answer that does not arrive within five seconds + leaves the sessions for the next load rather than delete on what this browser guessed. That + route answers for a workspace the caller can be authed into, unlike the status: a status is + what to do with the caller's own sessions, a setting is the workspace's to tell, so a + disabled membership is told nothing though its sessions still reconcile. + + Each session's record goes before its pieces, so nothing plans a push for it afterwards, + and a localStorage key written before the record and removed once every piece is gone makes + a later sweep finish a deletion that failed, unless a restore brought the session back + since. The record is deleted without the tombstone a user delete leaves, which is what lets + a restore bring it back. The session's dirty mark and sync row go with it (`sessionSwept`), + unless the row still carries a removal or a restore's staging. Nothing is sent to the storage: the local + copy's age says nothing about another device's, which may have pushed the session since, + and the server applies the rule to the backup on its own. A session swept here that the + storage still lists comes back on the next restore. ## Limits diff --git a/frontend/src/lib/components/copilot/chat/HistoryManager.svelte.ts b/frontend/src/lib/components/copilot/chat/HistoryManager.svelte.ts index 4edac6dc0a..d956b4c0c0 100644 --- a/frontend/src/lib/components/copilot/chat/HistoryManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/HistoryManager.svelte.ts @@ -267,15 +267,22 @@ export async function importStoredChats( return true } -/** Deletes these chats of the session (with their images) and these images: what an earlier - * restore staged for it and the backup no longer has. False when nothing could be deleted. */ +/** Every chat tagged with the session, and their images: for a session past its workspace's + * retention, which no runtime has mounted. */ +export function deleteSessionChats(sessionId: string, email: string): Promise { + return pruneSessionChats(sessionId, undefined, new Set(), email) +} + +/** Deletes chats of the session (with their images) and these images: what an earlier restore + * staged for it and the backup no longer has. `chats` names the ones to go; undefined is every + * chat of the session. False when nothing could be deleted. */ export async function pruneSessionChats( sessionId: string, - chats: Set, + chats: Set | undefined, images: Set, email: string ): Promise { - if (chats.size === 0 && images.size === 0) return true + if (chats?.size === 0 && images.size === 0) return true const db = await backupDb(email) if (!db) return false try { @@ -283,7 +290,7 @@ export async function pruneSessionChats( const chatStore = tx.objectStore('chats') const imageStore = tx.objectStore('images') for (const chatId of await chatStore.index('by-session').getAllKeys(sessionId)) { - if (!chats.has(String(chatId))) continue + if (chats && !chats.has(String(chatId))) continue await chatStore.delete(chatId) const keys = await imageStore .index('by-chat') diff --git a/frontend/src/lib/components/copilot/chat/artifacts/artifactsDB.test.ts b/frontend/src/lib/components/copilot/chat/artifacts/artifactsDB.test.ts index d70566ad18..a0fb5a4a4b 100644 --- a/frontend/src/lib/components/copilot/chat/artifacts/artifactsDB.test.ts +++ b/frontend/src/lib/components/copilot/chat/artifacts/artifactsDB.test.ts @@ -220,7 +220,7 @@ describe('artifactsDB', () => { expect(await noDb.getArtifact('a1')).toBeUndefined() expect(await noDb.listArtifactsForSession('s1')).toEqual([]) await expect(noDb.deleteArtifact('a1')).resolves.toBeUndefined() - await expect(noDb.deleteArtifactsForSession('s1')).resolves.toBeUndefined() + await expect(noDb.deleteArtifactsForSession('s1')).resolves.toBe(false) }) it('rejects a version read it could not make, instead of reading as absent', async () => { diff --git a/frontend/src/lib/components/copilot/chat/artifacts/artifactsDB.ts b/frontend/src/lib/components/copilot/chat/artifacts/artifactsDB.ts index e7ff4b1f60..f0cb32d71f 100644 --- a/frontend/src/lib/components/copilot/chat/artifacts/artifactsDB.ts +++ b/frontend/src/lib/components/copilot/chat/artifacts/artifactsDB.ts @@ -449,9 +449,14 @@ export async function pruneSessionArtifacts( } } -export async function deleteArtifactsForSession(sessionId: string): Promise { +/** False when the store could not be reached or the deletion failed. With `email`, only that + * user's store is touched: a caller that captured its user must not follow an account switch. */ +export async function deleteArtifactsForSession( + sessionId: string, + email?: string +): Promise { const db = await getDB() - if (!db) return + if (!db || (email !== undefined && db.name !== scopedKeyFor(ARTIFACTS_DB, email))) return false try { const tx = db.transaction(['items', 'versions'], 'readwrite') const items = tx.objectStore('items') @@ -464,8 +469,10 @@ export async function deleteArtifactsForSession(sessionId: string): Promise { putItem({ id: 'a', sessionId: 's1', kind: 'snapshot', name: 'x.txt', addedAt: 0 }) ).resolves.toBeUndefined() await expect(deleteItem('a')).resolves.toBeUndefined() - await expect(deleteItemsForSession('s1')).resolves.toBeUndefined() + await deleteItemsForSession('s1') }) it('does not throw when requesting persistent storage', async () => { diff --git a/frontend/src/lib/components/copilot/chat/files/attachedFilesDB.ts b/frontend/src/lib/components/copilot/chat/files/attachedFilesDB.ts index fe051ef8cc..dc872845cd 100644 --- a/frontend/src/lib/components/copilot/chat/files/attachedFilesDB.ts +++ b/frontend/src/lib/components/copilot/chat/files/attachedFilesDB.ts @@ -91,9 +91,10 @@ export async function deleteItem(id: string): Promise { await db?.delete('items', id) } -export async function deleteItemsForSession(sessionId: string): Promise { +/** False when the store could not be reached or the deletion failed. */ +export async function deleteItemsForSession(sessionId: string): Promise { const db = await getDB() - if (!db) return + if (!db) return false try { const tx = db.transaction('items', 'readwrite') const index = tx.store.index('by-session') @@ -103,8 +104,10 @@ export async function deleteItemsForSession(sessionId: string): Promise { cursor = await cursor.continue() } await tx.done + return true } catch (err) { console.error('Could not delete attached files for session', err) + return false } } diff --git a/frontend/src/lib/components/sessions/sessionMirror.svelte.ts b/frontend/src/lib/components/sessions/sessionMirror.svelte.ts index a18e2febd6..bfa1621822 100644 --- a/frontend/src/lib/components/sessions/sessionMirror.svelte.ts +++ b/frontend/src/lib/components/sessions/sessionMirror.svelte.ts @@ -27,7 +27,7 @@ import { getCurrentUserEmail, onUserChange, scopedKey, scopedKeyFor } from '$lib import { logFeatureUsage } from '$lib/utils/featureUsage' import { randomUUID } from '$lib/utils/uuid' import { workspaceRootId } from './sessionScope.svelte' -import { onMirrorSignal } from './sessionMirrorSignal' +import { onMirrorSignal, onSessionSwept, sessionsLockName } from './sessionMirrorSignal' import { importSessions, isSessionTombstoned, @@ -385,7 +385,7 @@ function hasWebLocks(): boolean { async function withUserLock(email: string, fn: () => Promise, wait = false): Promise { const locks = webLocks() if (!locks) return fn() - await locks.request(`wm-ai-sessions-mirror::${email}`, { ifAvailable: !wait }, async (lock) => { + await locks.request(sessionsLockName(email), { ifAvailable: !wait }, async (lock) => { if (lock) await fn() // The other tab's flush read the marks before this one's were written: try again // once it is done, rather than wait for the next write or load. @@ -1572,6 +1572,14 @@ export function backupSettingsChanged(ws: string): void { // --- Wiring --- if (BROWSER) { + // Nothing pushes a swept session again, so its mark and sync row are dead weight; a row + // still carrying a removal or a restore's staging is left to those. + onSessionSwept(async (id, email) => { + if (email !== getCurrentUserEmail()) return + dropDirty(id) + const row = await readSync(id, email) + if (row && !row.removed && !row.staging) await deleteSync([id], email) + }) onMirrorSignal((signal) => { // A mark for another user waits for that user's next load. const mine = !signal.email || signal.email === getCurrentUserEmail() diff --git a/frontend/src/lib/components/sessions/sessionMirror.test.ts b/frontend/src/lib/components/sessions/sessionMirror.test.ts index ff2c8373da..7676662d31 100644 --- a/frontend/src/lib/components/sessions/sessionMirror.test.ts +++ b/frontend/src/lib/components/sessions/sessionMirror.test.ts @@ -93,7 +93,7 @@ import { sessionState, type Session } from './sessionState.svelte' -import { markSessionDirty } from './sessionMirrorSignal' +import { markSessionDirty, sessionSwept } from './sessionMirrorSignal' import { __flushForTesting, __resetMirrorForTesting, @@ -268,6 +268,19 @@ describe('sessionMirror flush', () => { await __settleForTesting() }) + it('forgets the sync row of a session the retention swept, unless it carries a removal', async () => { + await __writeSyncForTesting( + [ + { id: 'swept', ws: 'admins', head: '', chats: {}, images: {} }, + { id: 'swept-removed', ws: 'admins', head: '', chats: {}, images: {}, removed: true } + ], + EMAIL + ) + await sessionSwept('swept', EMAIL) + await sessionSwept('swept-removed', EMAIL) + expect((await __syncRowsForTesting(EMAIL)).map((r) => r.id)).toEqual(['swept-removed']) + }) + it('keeps a delete filed on the sync row while the first push is still in flight', async () => { const s: Session = { id: 'sr', name: 'session-1', createdAt: 1, workspace_id: 'ws' } sessionState.sessions = [s] diff --git a/frontend/src/lib/components/sessions/sessionMirrorPlan.ts b/frontend/src/lib/components/sessions/sessionMirrorPlan.ts index a664e61f5e..caea48c792 100644 --- a/frontend/src/lib/components/sessions/sessionMirrorPlan.ts +++ b/frontend/src/lib/components/sessions/sessionMirrorPlan.ts @@ -79,9 +79,9 @@ export function isFallbackStorage(name: string): boolean { /** * The part of a session record the backup keeps. Left out on purpose: `name` (a * per-browser counter), the unsent-draft fields (`pending_*`, `draftPrompt`, - * `autoSendDraftAt`), `workspace_root_id` (derived on import), `transient`, and the two - * fields reading a session bumps (`lastSeenCount`, `lastActivityAt`) — so opening a - * session and reading its new messages never costs a push. + * `autoSendDraftAt`), `workspace_root_id` (derived on import), `transient`, `restoredAt` + * (this browser's clock), and the two fields reading a session bumps (`lastSeenCount`, + * `lastActivityAt`) — so opening a session and reading its new messages never costs a push. */ export type SessionHead = Pick< Session, diff --git a/frontend/src/lib/components/sessions/sessionMirrorSignal.ts b/frontend/src/lib/components/sessions/sessionMirrorSignal.ts index eb936d2e22..abe600b4a6 100644 --- a/frontend/src/lib/components/sessions/sessionMirrorSignal.ts +++ b/frontend/src/lib/components/sessions/sessionMirrorSignal.ts @@ -29,6 +29,13 @@ export function markSessionRemoved(sessionId: string, workspaceId?: string, emai emit({ kind: 'removed', sessionId, workspaceId, email }) } +/** The Web Lock one tab of the user holds while it reads or writes the stores wholesale: the + * backup's flush and restore, and the retention sweep, which must not interleave with either + * (a flush planning a session half deleted would push the deletions to the backup). */ +export function sessionsLockName(email: string): string { + return `wm-ai-sessions-mirror::${email}` +} + export function onMirrorSignal(fn: (signal: MirrorSignal) => void): void { handler = fn const replay = buffered @@ -36,7 +43,24 @@ export function onMirrorSignal(fn: (signal: MirrorSignal) => void): void { for (const signal of replay) fn(signal) } +let sweptHandler: ((sessionId: string, email: string) => Promise) | undefined + +/** The retention sweep deleted this session's local copy in the store of `email`: what the + * backup keeps of it in this browser goes too. Awaited under the sweep's tab lock. */ +export async function sessionSwept(sessionId: string, email: string): Promise { + try { + await sweptHandler?.(sessionId, email) + } catch (e) { + console.error('Could not forget the backup state of a swept session', e) + } +} + +export function onSessionSwept(fn: (sessionId: string, email: string) => Promise): void { + sweptHandler = fn +} + export function __resetMirrorSignalForTesting(): void { handler = undefined + sweptHandler = undefined buffered = [] } diff --git a/frontend/src/lib/components/sessions/sessionState.svelte.ts b/frontend/src/lib/components/sessions/sessionState.svelte.ts index 2670b3fa4d..3014e6d5e7 100644 --- a/frontend/src/lib/components/sessions/sessionState.svelte.ts +++ b/frontend/src/lib/components/sessions/sessionState.svelte.ts @@ -27,7 +27,13 @@ import { userScopedDb } from '$lib/userScopedDb' import { emailOfScopedKey, scopedKeyFor } from '$lib/userScopedStorage' import { deleteItemsForSession } from '../copilot/chat/files/attachedFilesDB' import { deleteArtifactsForSession } from '../copilot/chat/artifacts/artifactsDB' -import { markSessionDirty, markSessionRemoved } from './sessionMirrorSignal' +import { deleteSessionChats } from '../copilot/chat/HistoryManager.svelte' +import { + markSessionDirty, + markSessionRemoved, + sessionSwept, + sessionsLockName +} from './sessionMirrorSignal' // Switch the global workspace iff the target differs from the active one // and is non-empty. Centralises the "session needs its workspace in focus" @@ -124,6 +130,11 @@ export type Session = { // Absent on records last written before the field existed; readers fall back // to createdAt via sessionLastActivityAt. lastActivityAt?: number + // When this browser restored the session from its backup, by this browser's clock. + // The restore sets `lastActivityAt` to the backup's time, the storage's clock; the + // retention counts from whichever is later, so a browser clock ahead of the storage's + // never deletes a session it just brought back. Not backed up. + restoredAt?: number // Per-session unread watermark: the displayMessages count the last time // the user was on this session's page. Compared against the runtime's // current message count to derive the unread badge (see sessionUnread). @@ -438,8 +449,15 @@ export function __resetDeletedSessionIdsForTesting(): void { // The one way to remove a session's record. Tombstones BEFORE awaiting the delete so a // putSession racing this transaction cannot commit its write behind it — a direct // db.delete elsewhere would silently reopen that window. -async function deleteSessionRow(db: IDBPDatabase, id: string): Promise { - deletedSessionIds.add(id) +async function deleteSessionRow( + db: IDBPDatabase, + id: string, + // The retention sweep passes false. It holds the in-use lock exclusively, so no write can + // race its delete, and the backup may still hold the session: a tombstone would refuse the + // restore that is meant to bring it back. + tombstone = true +): Promise { + if (tombstone) deletedSessionIds.add(id) await db.delete('sessions', id) } @@ -634,6 +652,242 @@ export async function reconcileSessionsLifecycle(): Promise { } } +// --- Retention --- + +const DAY_MS = 24 * 60 * 60 * 1000 + +// Past the retention by this browser's clock, counted from the later of the session's last +// activity and its restore here: a restored session carries the backup's time, the storage's +// clock, so without `restoredAt` a browser running ahead would delete what it just brought +// back. Archived sessions count like any other. +function isSessionExpired( + session: Session, + retentionDays: number | undefined, + now: number +): boolean { + if (retentionDays === undefined || !(retentionDays >= 1)) return false + const since = Math.max(sessionLastActivityAt(session), session.restoredAt ?? 0) + return since < now - retentionDays * DAY_MS +} + +// What the server last told this browser, and when. It decides whether the sweep asks again, +// and nothing else: a retention raised or cleared since must not delete a session, and a +// persisted unsent draft has no backup to come back from. +const RETENTION_DAYS = 'windmill_sessions_retention_days' + +// Nothing remembered for longer than this is trusted even to say there is nothing to ask +// about, so a retention lowered while this browser saw nothing expiring still takes effect. +const RETENTION_STALE_MS = 24 * 60 * 60 * 1000 + +interface RememberedRetention { + at: number + days: Record +} + +function rememberRetention(email: string, days: Record): void { + try { + const remembered: RememberedRetention = { at: Date.now(), days } + localStorage.setItem(scopedKeyFor(RETENTION_DAYS, email), JSON.stringify(remembered)) + } catch {} +} + +function rememberedRetention(email: string): RememberedRetention | undefined { + try { + const stored = localStorage.getItem(scopedKeyFor(RETENTION_DAYS, email)) + const remembered = stored ? JSON.parse(stored) : undefined + if (remembered?.days && typeof remembered.days === 'object') { + return remembered as RememberedRetention + } + } catch {} + return undefined +} + +// How long the sweep waits for the retention of the workspaces it is about to sweep in. The +// tab reads its sessions after the sweep, so a request nothing answers costs the list this +// much and no more, and only in a tab that had something to delete. +const RETENTION_ASK_MS = 5000 + +// The retention the server gives now, or undefined when this browser could not be told: a +// session is deleted only on an answer of the moment. +async function askRetention(workspaceIds: string[]): Promise | undefined> { + try { + return await Promise.race([ + WorkspaceService.getSessionWorkspaceRetention({ + requestBody: { workspace_ids: workspaceIds } + }), + new Promise((resolve) => setTimeout(() => resolve(undefined), RETENTION_ASK_MS)) + ]) + } catch (e) { + console.error('Failed to read the AI session retention of the workspaces', e) + return undefined + } +} + +// One key per session this browser swept whose pieces are not all deleted yet. +const RETENTION_PENDING = 'windmill_sessions_retention_pending' + +function retentionPendingPrefix(email: string): string { + return `${scopedKeyFor(RETENTION_PENDING, email)}::` +} + +function forgetRetentionPending(email: string, id: string): void { + try { + localStorage.removeItem(retentionPendingPrefix(email) + id) + } catch {} +} + +function retentionPending(email: string): string[] { + const prefix = retentionPendingPrefix(email) + const ids: string[] = [] + try { + for (let i = 0; i < localStorage.length; i++) { + const key = localStorage.key(i) + if (key?.startsWith(prefix)) ids.push(key.slice(prefix.length)) + } + } catch {} + return ids +} + +function webLocks(): LockManager | undefined { + return typeof navigator === 'undefined' ? undefined : (navigator as { locks?: LockManager }).locks +} + +// Held, shared, by every tab from before it reads the user's sessions until it stops using +// them: the stores are shared, and each tab keeps copies of the sessions in memory, so the +// sweep deletes only while holding this exclusively. +function sessionsInUseLockName(email: string): string { + return `${sessionsLockName(email)}::in-use` +} + +interface InUseHold { + email: string + released: boolean + release?: () => void + done?: Promise +} + +let inUse: InUseHold | undefined + +// Resolves once the hold is granted, which waits for the sweep another tab is running. A +// request the browser refuses (a document that is not fully active) resolves it too, without +// a hold: the tab reads its sessions unguarded, as it does where Web Locks do not exist, and +// never sits waiting for a grant that is not coming. +async function holdSessionsInUse(email: string): Promise { + const locks = webLocks() + if (!locks || inUse?.email === email) return + await releaseSessionsInUse() + const hold: InUseHold = { email, released: false } + inUse = hold + await new Promise((granted) => { + hold.done = locks + .request(sessionsInUseLockName(email), { mode: 'shared' }, () => { + granted() + return hold.released ? undefined : new Promise((resolve) => (hold.release = resolve)) + }) + .catch((e) => { + console.error('Could not hold the AI sessions this tab is reading', e) + if (inUse === hold) inUse = undefined + granted() + }) + }) +} + +// Resolves once the hold is let go of, so an exclusive request made next can be granted. +async function releaseSessionsInUse(): Promise { + const hold = inUse + if (!hold) return + inUse = undefined + hold.released = true + hold.release?.() + await hold.done?.catch(() => {}) +} + +// Deletes one expired session: its record first, so nothing plans a push for it afterwards, +// then its pieces. The pending key, written before the record and removed once every piece +// is gone, is what a later sweep finishes a failed deletion from. +async function sweepSession( + db: IDBPDatabase, + id: string, + email: string +): Promise { + try { + localStorage.setItem(retentionPendingPrefix(email) + id, '1') + } catch { + return + } + await deleteSessionRow(db, id, false) + await sessionSwept(id, email) + if (await deleteSessionPieces(id, email)) forgetRetentionPending(email, id) +} + +// Chats with their images, artifacts and attached files. False when any of them could not +// be deleted. +async function deleteSessionPieces(id: string, email: string): Promise { + const chats = await deleteSessionChats(id, email) + const artifacts = await deleteArtifactsForSession(id, email) + const files = await deleteItemsForSession(id) + return chats && artifacts && files +} + +// The workspace a session's retention comes from: persisted unsent drafts count by the one +// they are waiting on. +function retentionWorkspaceOf(session: Session): string | undefined { + return session.workspace_id ?? session.pending_workspace_id +} + +// Deletes this browser's copies of the sessions past their workspace's retention, and the +// pieces of the ones an earlier sweep could not finish (docs/ai-session-backups.md). Deleting +// one record at a time, without re-reading it, is safe only under the in-use lock held +// exclusively, granted exactly when no tab has the sessions loaded — hence the call site. +async function sweepExpiredSessions(email: string): Promise { + const locks = webLocks() + if (!locks || inUse) return + try { + await locks.request(sessionsInUseLockName(email), { ifAvailable: true }, async (idle) => { + if (!idle) return + // The flush and the restore run under this one: neither must see a session half + // deleted, or plan a push from it. + await locks.request(sessionsLockName(email), { ifAvailable: true }, async (mirror) => { + if (!mirror) return + const db = await sessionsDb.whenReady() + if (!db || db.name !== scopedKeyFor(SESSIONS_DB, email)) return + for (const id of retentionPending(email)) { + // A restore brought the session back: its pieces are that copy's now. + const back = (await db.getKey('sessions', id)) !== undefined + if (back || (await deleteSessionPieces(id, email))) forgetRetentionPending(email, id) + } + const stored = await db.getAll('sessions') + const remembered = rememberedRetention(email) + const now = Date.now() + const workspaces = new Set() + let expired = false + for (const s of stored) { + const ws = retentionWorkspaceOf(s) + if (ws === undefined) continue + workspaces.add(ws) + expired ||= isSessionExpired(s, remembered?.days[ws], now) + } + // Nothing to sweep in, or nothing old enough by an answer recent enough to be + // believed about that: this load costs no request. + const fresh = remembered !== undefined && now - remembered.at < RETENTION_STALE_MS + if (workspaces.size === 0 || (!expired && fresh)) return + const retention = await askRetention([...workspaces]) + // Asked and not told: the sessions wait for the next load rather than go on an + // answer this browser does not have. + if (!retention) return + rememberRetention(email, retention) + for (const s of stored) { + const ws = retentionWorkspaceOf(s) + if (ws === undefined || !isSessionExpired(s, retention[ws], Date.now())) continue + await sweepSession(db, s.id, email) + } + }) + }) + } catch (e) { + console.error('Failed to sweep the sessions past their retention', e) + } +} + // The single seam for "a workspace just changed — bring sessions back in sync." // Refresh the workspace list FIRST — both reconcile and the putSession guard // read it, so it must reflect the change before reconcile runs — then reconcile. @@ -715,6 +969,15 @@ export async function deleteSessionsForWorkspace(workspaceId: string): Promise { if (!BROWSER) return + // The retention sweep runs here and nowhere else: this tab holds none of the new user's + // sessions yet, and letting go of the hold it had leaves it holding none of anyone's. The + // new hold is taken before the sessions are read, so another tab's sweep never deletes + // what this tab is about to load, and one already running is waited for. + await releaseSessionsInUse() + if (email) { + await sweepExpiredSessions(email) + await holdSessionsInUse(email) + } await hydrateSessions({ dropTransients: prevEmail !== email }) // onUserChange also fires at registration time, before the email resolves — // that hydration is an empty no-op and must not clear the loading state. @@ -1197,9 +1460,10 @@ export async function importSessions(records: Session[], email: string): Promise const tx = db.transaction('sessions', 'readwrite') const existing = new Set((await tx.store.getAllKeys()).map(String)) let next = nextSessionNumber([...(await tx.store.getAll()), ...sessionState.sessions]) + const restoredAt = Date.now() for (const r of records) { if (existing.has(r.id) || deletedSessionIds.has(r.id)) continue - const record: Session = { ...r, name: `session-${next++}` } + const record: Session = { ...r, name: `session-${next++}`, restoredAt } delete record.transient delete record.workspace_root_id ensureSessionRootId(record) diff --git a/frontend/src/lib/components/sessions/sessionStateIndexedDb.test.ts b/frontend/src/lib/components/sessions/sessionStateIndexedDb.test.ts index b9882d045e..dddfe795f7 100644 --- a/frontend/src/lib/components/sessions/sessionStateIndexedDb.test.ts +++ b/frontend/src/lib/components/sessions/sessionStateIndexedDb.test.ts @@ -9,20 +9,30 @@ vi.mock('esm-env', async (importOriginal) => ({ })) // Spy on the attached-file GC so we can assert lifecycle deletes clean it up. -const { deleteItemsForSessionMock } = vi.hoisted(() => ({ deleteItemsForSessionMock: vi.fn() })) +const { deleteItemsForSessionMock } = vi.hoisted(() => ({ + deleteItemsForSessionMock: vi.fn().mockResolvedValue(true) +})) vi.mock('../copilot/chat/files/attachedFilesDB', async (orig) => ({ ...(await orig()), deleteItemsForSession: deleteItemsForSessionMock })) const { deleteArtifactsForSessionMock } = vi.hoisted(() => ({ - deleteArtifactsForSessionMock: vi.fn() + deleteArtifactsForSessionMock: vi.fn().mockResolvedValue(true) })) vi.mock('../copilot/chat/artifacts/artifactsDB', async (orig) => ({ ...(await orig()), deleteArtifactsForSession: deleteArtifactsForSessionMock })) +const { deleteSessionChatsMock } = vi.hoisted(() => ({ + deleteSessionChatsMock: vi.fn().mockResolvedValue(true) +})) +vi.mock('../copilot/chat/HistoryManager.svelte', async (orig) => ({ + ...(await orig()), + deleteSessionChats: deleteSessionChatsMock +})) + // sessionState imports WorkspaceService; these tests don't touch the network. vi.mock('$lib/gen', async (orig) => { const actual = await orig() @@ -31,7 +41,8 @@ vi.mock('$lib/gen', async (orig) => { WorkspaceService: { ...actual.WorkspaceService, listUserWorkspaces: vi.fn().mockResolvedValue([]), - getSessionWorkspaceStatus: vi.fn().mockResolvedValue({}) + getSessionWorkspaceStatus: vi.fn().mockResolvedValue({}), + getSessionWorkspaceRetention: vi.fn().mockResolvedValue({}) } } }) @@ -75,6 +86,32 @@ function freshUser() { return asUser(`u${n++}@x.com`) } +// The Web Locks API, which the node test environment lacks: `holders` counts the shared holds +// on each name across tabs, against which an exclusive request made if available is not granted. +function installLocks(holders: Map): void { + if (typeof navigator === 'undefined') { + Object.defineProperty(globalThis, 'navigator', { value: {}, configurable: true }) + } + Object.defineProperty(navigator, 'locks', { + value: { + request: async (name: string, ...rest: unknown[]) => { + const run = rest[rest.length - 1] as (lock: unknown) => Promise + const options = (rest.length > 1 ? rest[0] : {}) as LockOptions + if (options.mode === 'shared') { + holders.set(name, (holders.get(name) ?? 0) + 1) + try { + return await run({}) + } finally { + holders.set(name, (holders.get(name) ?? 1) - 1) + } + } + return run(options.ifAvailable && (holders.get(name) ?? 0) > 0 ? null : {}) + } + }, + configurable: true + }) +} + // Hydration is fire-and-forget off the user store, so it can land after the test body // has populated sessionState.sessions and overwrite it with what the DB held at read // time; `hydrated` flips once the read has been applied. The logout is load-bearing: @@ -679,6 +716,93 @@ describe('sessionState IndexedDB persistence', () => { deleteSession('draftRec') }) + it('sweeps sessions past their workspace retention when a tab loads alone', async () => { + const user = freshUser() + usersWorkspaceStore.set({ + email: user.email, + workspaces: [ + { id: 'kept-ws', name: 'kept', disabled: false }, + { id: 'other-ws', name: 'other', disabled: false } + ] as never + }) + // The sweep runs only where Web Locks exist, and only as a tab loads: `login` is one. + const holders = new Map() + installLocks(holders) + const inUse = `wm-ai-sessions-mirror::${user.email}::in-use` + const otherTab = (n: number) => holders.set(inUse, (holders.get(inUse) ?? 0) + n) + await login(user) + const day = 24 * 60 * 60 * 1000 + const old = Date.now() - 31 * day + const stale = (id: string, over: Partial = {}) => + session({ id, createdAt: old, lastActivityAt: old, workspace_id: 'kept-ws', ...over }) + // Archived or not, a session is judged by its own last activity; one read a day ago + // stays, as do one restored here a day ago whatever the backup's time and one in a + // workspace without retention. + await putSession(stale('stale')) + await putSession(stale('stale-archived', { archived: true })) + await putSession(stale('read-lately', { lastActivityAt: Date.now() - day })) + await putSession(stale('restored-lately', { restoredAt: Date.now() - day })) + await putSession(stale('elsewhere', { workspace_id: 'other-ws' })) + + const retentionMock = vi.mocked(WorkspaceService.getSessionWorkspaceRetention) + let told: Record = { 'kept-ws': 30 } + retentionMock.mockImplementation(async () => told as never) + // The sweep believes a remembered answer for a day, so ageing it is how a later load + // is made to ask again. + const forgetWhenAsked = () => { + const key = `windmill_sessions_retention_days::${user.email}` + const remembered = JSON.parse(localStorage.getItem(key) ?? '{}') + localStorage.setItem(key, JSON.stringify({ ...remembered, at: Date.now() - 2 * day })) + } + const stored = async () => { + const db = await openDB(`windmill-sessions::${user.email}`, 1) + const ids = ((await db.getAll('sessions' as never)) as Session[]).map((s) => s.id) + db.close() + return ids.sort() + } + const chatDeletions = (id: string) => + deleteSessionChatsMock.mock.calls.filter(([sid, email]) => sid === id && email === user.email) + + // While another tab has the sessions loaded, nothing is swept. + otherTab(1) + await rehydrate(user) + expect(await stored()).toContain('stale') + expect(chatDeletions('stale')).toHaveLength(0) + otherTab(-1) + + // The retention is cleared when the sweep asks: what the server says then is what + // deletes, and a browser that remembered one deletes nothing on it. + told = {} + await rehydrate(user) + expect(await stored()).toContain('stale') + expect(chatDeletions('stale')).toHaveLength(0) + told = { 'kept-ws': 30 } + forgetWhenAsked() + + // The chats of the first expired session the sweep reaches, `stale` by key order, + // cannot be deleted this time. + deleteSessionChatsMock.mockResolvedValueOnce(false) + await rehydrate(user) + expect(await stored()).toEqual(['elsewhere', 'read-lately', 'restored-lately']) + const pending = (id: string) => + localStorage.getItem(`windmill_sessions_retention_pending::${user.email}::${id}`) + expect(chatDeletions('stale-archived')).toHaveLength(1) + expect(pending('stale')).toBe('1') + expect(pending('stale-archived')).toBeNull() + + // The next load finishes what that deletion left, with nothing else to sweep. + await rehydrate(user) + expect(pending('stale')).toBeNull() + expect(chatDeletions('stale')).toHaveLength(2) + + // A swept session is not tombstoned: the backup another device pushed to brings it back. + await importSessions([stale('stale')], user.email) + expect(await stored()).toContain('stale') + // Both are shared with the tests that follow, which expect neither. + retentionMock.mockResolvedValue({} as never) + Object.defineProperty(navigator, 'locks', { value: undefined, configurable: true }) + }) + it('clears the in-memory list on logout', async () => { const user = freshUser() await login(user) diff --git a/frontend/src/lib/components/workspaceSettings/AISettings.svelte b/frontend/src/lib/components/workspaceSettings/AISettings.svelte index a1dcc76758..14d27cab18 100644 --- a/frontend/src/lib/components/workspaceSettings/AISettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/AISettings.svelte @@ -693,7 +693,7 @@
    From e8078f2a963166b09849650424583f5dcfd28a84 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 16 Sep 2026 09:58:29 +0200 Subject: [PATCH 36/44] fix: dispatch workflow-as-code tasks from a deployed flow's inline step (#11146) * fix: dispatch workflow-as-code tasks from a deployed flow's inline step * fix: give a workflow-as-code task its own result-cache key * fix: key a cached workflow-as-code task on its name and arguments * fix: hash a cached workflow-as-code task's arguments like any job's * chore: regenerate system prompts for the task cache_ttl docs * fix: key a cached workflow-as-code task on its step key, not its name * fix: key a cached workflow-as-code task on a fingerprint of its code * fix: keep the task() doc attached to task() * fix: key a cached inline task on its step key and the workflow input * docs: cache_ttl has no effect on a taskFlow target --- backend/tests/bun_jobs.rs | 94 +++++++++++++++++++ backend/tests/fixtures/wac_flow_script.sql | 53 +++++++++++ backend/windmill-worker/src/bun_executor.rs | 64 +++++++++---- backend/windmill-worker/src/common.rs | 30 +++++- backend/windmill-worker/src/worker.rs | 2 +- backend/windmill-worker/src/worker_flow.rs | 3 +- cli/src/guidance/skills.gen.ts | 23 +++++ python-client/wmill/wmill/client.py | 8 ++ system_prompts/auto-generated/prompts.ts | 23 +++++ system_prompts/auto-generated/script.md | 8 ++ system_prompts/auto-generated/sdks/python.md | 8 ++ .../auto-generated/sdks/wac-python.md | 8 ++ .../auto-generated/sdks/wac-typescript.md | 7 ++ .../skills/write-script-python3/SKILL.md | 8 ++ .../skills/write-workflow-as-code/SKILL.md | 15 +++ typescript-client/client.ts | 7 ++ 16 files changed, 341 insertions(+), 20 deletions(-) create mode 100644 backend/tests/fixtures/wac_flow_script.sql diff --git a/backend/tests/bun_jobs.rs b/backend/tests/bun_jobs.rs index 590963b291..63d5341ba3 100644 --- a/backend/tests/bun_jobs.rs +++ b/backend/tests/bun_jobs.rs @@ -1209,6 +1209,100 @@ export function main() { Ok(()) } +/// A deployed flow runs an inline step as the `flow_node` its deploy rewrote it into, +/// a `FlowScript` job rather than the preview job the editor runs. A workflow-as-code +/// step's `task()` children must dispatch from that kind too, as re-runs of the same +/// node, or the step passes its editor test and fails once deployed. +/// +/// The step is cached: a child that shared the parent's result-cache key would hand +/// its own result (`10`) back to the parent on resume, in place of the workflow's. +#[sqlx::test(fixtures("base", "wac_flow_script"))] +async fn test_bun_wac_task_dispatch_from_flow_script(db: Pool) -> anyhow::Result<()> { + use windmill_common::flows::FlowNodeId; + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let node = FlowNodeId(3000000000000011); + let job = RunJob::from(JobPayload::FlowScript { + id: node, + path: "f/system/wac_flow_script/a".to_string(), + language: ScriptLang::Bun, + cache_ttl: Some(60), + cache_ignore_s3_path: None, + dedicated_worker: None, + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default(), + }) + .arg("n", serde_json::json!(5)) + .run_until_complete(&db, false, port) + .await; + + assert_eq!( + job.json_result().unwrap(), + serde_json::json!({"doubled": 10}) + ); + + let children: Vec<(String, Option, Option)> = sqlx::query_as( + "SELECT kind::text, runnable_id, cache_ttl FROM v2_job WHERE parent_job = $1", + ) + .bind(job.id) + .fetch_all(&db) + .await?; + assert_eq!( + children, + vec![("flowscript".to_string(), Some(node.0), None)], + "the task child re-runs the parent's flow node, outside the result cache" + ); + Ok(()) +} + +/// `task(fn, { cache_ttl })` on an inline task of a deployed flow's step: the child runs +/// the parent's code with the parent's arguments, so its result-cache key carries its +/// step key, or the parent and every sibling would read its result back as their own. +#[sqlx::test(fixtures("base", "wac_flow_script"))] +async fn test_bun_wac_inline_task_cache_is_per_task(db: Pool) -> anyhow::Result<()> { + use windmill_common::flows::FlowNodeId; + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let payload = || JobPayload::FlowScript { + id: FlowNodeId(3000000000000012), + path: "f/system/wac_flow_script/a".to_string(), + language: ScriptLang::Bun, + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default(), + }; + + let mut children_from_cache = Vec::new(); + for _ in 0..2 { + let job = RunJob::from(payload()) + .arg("n", serde_json::json!(5)) + .run_until_complete(&db, false, port) + .await; + assert_eq!( + job.json_result().unwrap(), + serde_json::json!({"doubled": 10, "tripled": 15}) + ); + let from_cache: i64 = sqlx::query_scalar( + "SELECT count(*) FROM job_logs l JOIN v2_job j ON j.id = l.job_id \ + WHERE j.parent_job = $1 AND l.logs LIKE '%found in cache%'", + ) + .bind(job.id) + .fetch_one(&db) + .await?; + children_from_cache.push(from_cache); + } + assert_eq!( + children_from_cache, + vec![0, 2], + "the second run serves each task from its own cache entry" + ); + Ok(()) +} + // ============================================================================ // Environment Variable Tests // ============================================================================ diff --git a/backend/tests/fixtures/wac_flow_script.sql b/backend/tests/fixtures/wac_flow_script.sql new file mode 100644 index 0000000000..1b780c6cf4 --- /dev/null +++ b/backend/tests/fixtures/wac_flow_script.sql @@ -0,0 +1,53 @@ +-- A deployed flow whose inline bun step is workflow-as-code calling task(), in the +-- shape the deploy leaves behind: the RawScript module rewritten into a flow_node that +-- the step then runs as a FlowScript job. No lock, so the worker resolves +-- windmill-client at run time like the other bun fixtures. +INSERT INTO public.flow(workspace_id, summary, description, path, versions, schema, value, edited_by) VALUES ( +'test-workspace', '', '', +'f/system/wac_flow_script', +'{}', +'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"n":{"type":"integer","description":""}},"required":[],"type":"object"}', +'{"modules":[{"id":"a","value":{"type":"flowscript","id":3000000000000011,"language":"bun","input_transforms":{"n":{"expr":"flow_input.n","type":"javascript"}}}}]}', +'system' +); + +INSERT INTO public.flow_node(id, workspace_id, path, hash_v2, lock, code) VALUES ( +3000000000000011, +'test-workspace', +'f/system/wac_flow_script', +'0000000000000000000000000000000000000000000000000000000000000011', +NULL, +E'import { workflow, task } from "windmill-client"; + +const double = task(async (n: number) => { + return n * 2; +}); + +export const main = workflow(async (n: number) => { + const d = await double(n); + return { doubled: d }; +});' +); + +-- The same flow's step with two tasks that cache their own result. +INSERT INTO public.flow_node(id, workspace_id, path, hash_v2, lock, code) VALUES ( +3000000000000012, +'test-workspace', +'f/system/wac_flow_script', +'0000000000000000000000000000000000000000000000000000000000000012', +NULL, +E'import { workflow, task } from "windmill-client"; + +const double = task(async (n: number) => { + return n * 2; +}, { cache_ttl: 60 }); +const triple = task(async (n: number) => { + return n * 3; +}, { cache_ttl: 60 }); + +export const main = workflow(async (n: number) => { + const d = await double(n); + const t = await triple(n); + return { doubled: d, tripled: t }; +});' +); diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index 7d45cab178..117fbda086 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -2874,15 +2874,17 @@ pub async fn handle_wac_v2_output( .collect(); // Resolve job_payload once (same for all children since they re-run - // the parent script) + // the parent script). The step's cache setting is for the workflow's + // result; a task is cached only through its own `cache_ttl` option, + // under a key of its own (see `cached_result_path`). let job_payload_template = match job.kind { JobKind::Script => { if let Some(hash) = job.runnable_id { Ok(JobPayload::ScriptHash { hash, path: job.runnable_path.clone().unwrap_or_default(), - cache_ttl: job.cache_ttl, - cache_ignore_s3_path: job.cache_ignore_s3_path, + cache_ttl: None, + cache_ignore_s3_path: None, dedicated_worker: None, language: job.script_lang.unwrap_or(ScriptLang::Bun), priority: job.priority, @@ -2897,6 +2899,27 @@ pub async fn handle_wac_v2_output( )) } } + // A deployed flow runs an inline step as the `flow_node` its deploy + // rewrote it into; the child re-runs that node the way a `Script` + // child re-runs its hash, so `runnable_id` (the checkpoint's source + // hash) stays the same across parent and children. + JobKind::FlowScript => { + if let Some(id) = job.runnable_id { + Ok(JobPayload::FlowScript { + id: windmill_common::flows::FlowNodeId(id.0), + path: job.runnable_path.clone().unwrap_or_default(), + language: job.script_lang.unwrap_or(ScriptLang::Bun), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + concurrency_settings: ConcurrencySettings::default(), + }) + } else { + Err(error::Error::internal_err( + "WAC v2 FlowScript job missing runnable_id".to_string(), + )) + } + } JobKind::Preview => { let row: Option<(Option, Option)> = sqlx::query_as( "SELECT raw_code, raw_lock FROM v2_job WHERE id = $1 AND workspace_id = $2", @@ -2912,8 +2935,8 @@ pub async fn handle_wac_v2_output( hash: None, language: job.script_lang.unwrap_or(ScriptLang::Bun), lock: lock, - cache_ttl: job.cache_ttl, - cache_ignore_s3_path: job.cache_ignore_s3_path, + cache_ttl: None, + cache_ignore_s3_path: None, dedicated_worker: None, concurrency_settings: ConcurrencySettingsWithCustom::default(), debouncing_settings: DebouncingSettings::default(), @@ -3012,6 +3035,12 @@ pub async fn handle_wac_v2_output( let mut pushed_ids: Vec = Vec::with_capacity(num_steps); let push_result: error::Result<()> = async { for (step, (_, child_uuid)) in steps.iter().zip(job_ids.iter()) { + // A task with a runnable of its own (a deployed script or flow) queues + // at that runnable's priority; any other task is the parent's code and + // queues at the parent's. + let own_runnable = matches!(step.dispatch_type.as_str(), "script" | "flow") + && !step.script.starts_with("./"); + // Resolve job payload based on dispatch_type let (job_payload, child_args, is_external, on_behalf_of) = match step.dispatch_type.as_str() { @@ -3025,8 +3054,8 @@ pub async fn handle_wac_v2_output( hash: None, language: module.language, lock: module.lock, - cache_ttl: job.cache_ttl, - cache_ignore_s3_path: job.cache_ignore_s3_path, + cache_ttl: None, + cache_ignore_s3_path: None, dedicated_worker: None, concurrency_settings: ConcurrencySettingsWithCustom::default(), debouncing_settings: DebouncingSettings::default(), @@ -3110,7 +3139,8 @@ pub async fn handle_wac_v2_output( let mut job_payload = job_payload; if let Some(cache_ttl) = step.cache_ttl { match &mut job_payload { - JobPayload::ScriptHash { cache_ttl: ref mut ct, .. } => { + JobPayload::ScriptHash { cache_ttl: ref mut ct, .. } + | JobPayload::FlowScript { cache_ttl: ref mut ct, .. } => { *ct = Some(cache_ttl) } JobPayload::Code(ref mut code) => code.cache_ttl = Some(cache_ttl), @@ -3122,7 +3152,8 @@ pub async fn handle_wac_v2_output( || step.concurrency_time_window_s.is_some() { match &mut job_payload { - JobPayload::ScriptHash { concurrency_settings: ref mut cs, .. } => { + JobPayload::ScriptHash { concurrency_settings: ref mut cs, .. } + | JobPayload::FlowScript { concurrency_settings: ref mut cs, .. } => { if let Some(limit) = step.concurrent_limit { cs.concurrent_limit = Some(limit); } @@ -3188,13 +3219,14 @@ pub async fn handle_wac_v2_output( job.visible_to_owner, step.tag.clone().or_else(|| Some(job.tag.clone())), step.timeout.or(job.timeout), - None, // flow_step_id - step.priority, // priority_override - None, // authed - false, // running - None, // end_user_email - None, // trigger - None, // suspended_mode + None, // flow_step_id + step.priority + .or(if own_runnable { None } else { job.priority }), + None, // authed + false, // running + None, // end_user_email + None, // trigger + None, // suspended_mode ) .await?; diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index 0a5deb52d2..9c012fd117 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -1559,7 +1559,7 @@ pub async fn cached_result_path( client: &AuthedClient, job: &MiniPulledJob, raw_data: Option<&RawData>, -) -> String { +) -> windmill_common::error::Result { let mut hasher = sha2::Sha256::new(); hasher.update(&[job.kind as u8]); if let Some(ScriptHash(hash)) = job.runnable_id { @@ -1574,6 +1574,13 @@ pub async fn cached_result_path( _ => {} } } + // A workflow-as-code task child runs its parent's code with the parent's + // arguments; the step it executes is what tells its result from the parent's + // and from its siblings'. + if let Some(step_key) = wac_executing_key(db, job).await? { + hasher.update(b"wac_step:"); + hasher.update(step_key.as_bytes()); + } hash_args( db, client, @@ -1584,7 +1591,26 @@ pub async fn cached_result_path( job.cache_ignore_s3_path.unwrap_or(false), ) .await; - format!("g/results/{:064x}", hasher.finalize()) + Ok(format!("g/results/{:064x}", hasher.finalize())) +} + +/// The checkpoint step key a workflow-as-code parent seeded for this child at push +/// time; `None` for any job that is not such a child. +async fn wac_executing_key( + db: &DB, + job: &MiniPulledJob, +) -> windmill_common::error::Result> { + if job.parent_job.is_none() || job.flow_step_id.is_some() { + return Ok(None); + } + let key: Option> = sqlx::query_scalar( + "SELECT workflow_as_code_status->'_checkpoint'->>'_executing_key' \ + FROM v2_job_status WHERE id = $1", + ) + .bind(job.id) + .fetch_optional(db) + .await?; + Ok(key.flatten()) } #[cfg(feature = "parquet")] diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 467b8dbf19..56b4b26fb0 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -4645,7 +4645,7 @@ pub async fn handle_queued_job( let cached_res_path = if job.cache_ttl.is_some() { match conn { Connection::Sql(db) => { - Some(cached_result_path(db, &client, &job, preview_data.as_ref()).await) + Some(cached_result_path(db, &client, &job, preview_data.as_ref()).await?) } Connection::Http(_) => None, } diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index cbda50c7bc..ed543bd517 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -2005,7 +2005,8 @@ pub async fn update_flow_status_after_job_completion_internal( if flow_job.cache_ttl.is_some() && success { let flow = RawData::Flow(flow_data.clone()); - let cached_res_path = cached_result_path(db, client, &flow_job, Some(&flow)).await; + let cached_res_path = + cached_result_path(db, client, &flow_job, Some(&flow)).await?; save_in_cache( db, diff --git a/cli/src/guidance/skills.gen.ts b/cli/src/guidance/skills.gen.ts index 60fdd07124..f58092fdac 100644 --- a/cli/src/guidance/skills.gen.ts +++ b/cli/src/guidance/skills.gen.ts @@ -4608,6 +4608,14 @@ def parse_sql_client_name(name: str) -> tuple[str, Optional[str]] # it grows with both the width of the fan-out and \`\`attempts\`\`. Retries with # no \`\`delay\`\` all go out in a single round. # +# \`\`cache_ttl\`\` serves a previous result of the task for that many seconds +# instead of running it again. A task is keyed on its step key (its name and +# call order) and the workflow's input, not on the arguments it is called +# with, so cache one only when whether it runs, and what it receives, follow +# from the workflow's input alone. A \`\`task_script\`\` target is keyed on the +# arguments it is called with. It has no effect on a \`\`task_flow\`\` target, +# which keeps its flow's own cache policy. +# # Usage:: # # @task @@ -6742,6 +6750,13 @@ export interface TaskRetry { export interface TaskOptions { timeout?: number; tag?: string; + /** Seconds during which a previous result of this task is served instead of + * running it again. A task written inline in the workflow is keyed on its + * step key (its name and call order) and the workflow's input, not on the + * arguments it is called with, so cache one only when whether it runs, and + * what it receives, follow from the workflow's input alone. A \`taskScript\` + * target is keyed on the arguments it is called with. It has no effect on a + * \`taskFlow\` target, which keeps its flow's own cache policy. */ cache_ttl?: number; priority?: number; concurrency_limit?: number; @@ -6933,6 +6948,14 @@ def get_resume_urls(approver: str = None, flow_level: bool = None) -> dict # it grows with both the width of the fan-out and \`\`attempts\`\`. Retries with # no \`\`delay\`\` all go out in a single round. # +# \`\`cache_ttl\`\` serves a previous result of the task for that many seconds +# instead of running it again. A task is keyed on its step key (its name and +# call order) and the workflow's input, not on the arguments it is called +# with, so cache one only when whether it runs, and what it receives, follow +# from the workflow's input alone. A \`\`task_script\`\` target is keyed on the +# arguments it is called with. It has no effect on a \`\`task_flow\`\` target, +# which keeps its flow's own cache policy. +# # Usage:: # # @task diff --git a/python-client/wmill/wmill/client.py b/python-client/wmill/wmill/client.py index 918a16f2b0..dbc9d9b193 100644 --- a/python-client/wmill/wmill/client.py +++ b/python-client/wmill/wmill/client.py @@ -3327,6 +3327,14 @@ def task( it grows with both the width of the fan-out and ``attempts``. Retries with no ``delay`` all go out in a single round. + ``cache_ttl`` serves a previous result of the task for that many seconds + instead of running it again. A task is keyed on its step key (its name and + call order) and the workflow's input, not on the arguments it is called + with, so cache one only when whether it runs, and what it receives, follow + from the workflow's input alone. A ``task_script`` target is keyed on the + arguments it is called with. It has no effect on a ``task_flow`` target, + which keeps its flow's own cache policy. + Usage:: @task diff --git a/system_prompts/auto-generated/prompts.ts b/system_prompts/auto-generated/prompts.ts index 2a20e8f779..1e92c8e69b 100644 --- a/system_prompts/auto-generated/prompts.ts +++ b/system_prompts/auto-generated/prompts.ts @@ -2588,6 +2588,14 @@ def parse_sql_client_name(name: str) -> tuple[str, Optional[str]] # it grows with both the width of the fan-out and \`\`attempts\`\`. Retries with # no \`\`delay\`\` all go out in a single round. # +# \`\`cache_ttl\`\` serves a previous result of the task for that many seconds +# instead of running it again. A task is keyed on its step key (its name and +# call order) and the workflow's input, not on the arguments it is called +# with, so cache one only when whether it runs, and what it receives, follow +# from the workflow's input alone. A \`\`task_script\`\` target is keyed on the +# arguments it is called with. It has no effect on a \`\`task_flow\`\` target, +# which keeps its flow's own cache policy. +# # Usage:: # # @task @@ -2738,6 +2746,13 @@ export interface TaskRetry { export interface TaskOptions { timeout?: number; tag?: string; + /** Seconds during which a previous result of this task is served instead of + * running it again. A task written inline in the workflow is keyed on its + * step key (its name and call order) and the workflow's input, not on the + * arguments it is called with, so cache one only when whether it runs, and + * what it receives, follow from the workflow's input alone. A \`taskScript\` + * target is keyed on the arguments it is called with. It has no effect on a + * \`taskFlow\` target, which keeps its flow's own cache policy. */ cache_ttl?: number; priority?: number; concurrency_limit?: number; @@ -2929,6 +2944,14 @@ def get_resume_urls(approver: str = None, flow_level: bool = None) -> dict # it grows with both the width of the fan-out and \`\`attempts\`\`. Retries with # no \`\`delay\`\` all go out in a single round. # +# \`\`cache_ttl\`\` serves a previous result of the task for that many seconds +# instead of running it again. A task is keyed on its step key (its name and +# call order) and the workflow's input, not on the arguments it is called +# with, so cache one only when whether it runs, and what it receives, follow +# from the workflow's input alone. A \`\`task_script\`\` target is keyed on the +# arguments it is called with. It has no effect on a \`\`task_flow\`\` target, +# which keeps its flow's own cache policy. +# # Usage:: # # @task diff --git a/system_prompts/auto-generated/script.md b/system_prompts/auto-generated/script.md index 7883a8ba4b..f278a67fba 100644 --- a/system_prompts/auto-generated/script.md +++ b/system_prompts/auto-generated/script.md @@ -2728,6 +2728,14 @@ def parse_sql_client_name(name: str) -> tuple[str, Optional[str]] # it grows with both the width of the fan-out and ``attempts``. Retries with # no ``delay`` all go out in a single round. # +# ``cache_ttl`` serves a previous result of the task for that many seconds +# instead of running it again. A task is keyed on its step key (its name and +# call order) and the workflow's input, not on the arguments it is called +# with, so cache one only when whether it runs, and what it receives, follow +# from the workflow's input alone. A ``task_script`` target is keyed on the +# arguments it is called with. It has no effect on a ``task_flow`` target, +# which keeps its flow's own cache policy. +# # Usage:: # # @task diff --git a/system_prompts/auto-generated/sdks/python.md b/system_prompts/auto-generated/sdks/python.md index 4aacf4d1b4..4f727c7d45 100644 --- a/system_prompts/auto-generated/sdks/python.md +++ b/system_prompts/auto-generated/sdks/python.md @@ -672,6 +672,14 @@ def parse_sql_client_name(name: str) -> tuple[str, Optional[str]] # it grows with both the width of the fan-out and ``attempts``. Retries with # no ``delay`` all go out in a single round. # +# ``cache_ttl`` serves a previous result of the task for that many seconds +# instead of running it again. A task is keyed on its step key (its name and +# call order) and the workflow's input, not on the arguments it is called +# with, so cache one only when whether it runs, and what it receives, follow +# from the workflow's input alone. A ``task_script`` target is keyed on the +# arguments it is called with. It has no effect on a ``task_flow`` target, +# which keeps its flow's own cache policy. +# # Usage:: # # @task diff --git a/system_prompts/auto-generated/sdks/wac-python.md b/system_prompts/auto-generated/sdks/wac-python.md index 816ea4959b..a8e98a9d4f 100644 --- a/system_prompts/auto-generated/sdks/wac-python.md +++ b/system_prompts/auto-generated/sdks/wac-python.md @@ -58,6 +58,14 @@ def get_resume_urls(approver: str = None, flow_level: bool = None) -> dict # it grows with both the width of the fan-out and ``attempts``. Retries with # no ``delay`` all go out in a single round. # +# ``cache_ttl`` serves a previous result of the task for that many seconds +# instead of running it again. A task is keyed on its step key (its name and +# call order) and the workflow's input, not on the arguments it is called +# with, so cache one only when whether it runs, and what it receives, follow +# from the workflow's input alone. A ``task_script`` target is keyed on the +# arguments it is called with. It has no effect on a ``task_flow`` target, +# which keeps its flow's own cache policy. +# # Usage:: # # @task diff --git a/system_prompts/auto-generated/sdks/wac-typescript.md b/system_prompts/auto-generated/sdks/wac-typescript.md index 66eecfe761..608d75d18c 100644 --- a/system_prompts/auto-generated/sdks/wac-typescript.md +++ b/system_prompts/auto-generated/sdks/wac-typescript.md @@ -34,6 +34,13 @@ export interface TaskRetry { export interface TaskOptions { timeout?: number; tag?: string; + /** Seconds during which a previous result of this task is served instead of + * running it again. A task written inline in the workflow is keyed on its + * step key (its name and call order) and the workflow's input, not on the + * arguments it is called with, so cache one only when whether it runs, and + * what it receives, follow from the workflow's input alone. A `taskScript` + * target is keyed on the arguments it is called with. It has no effect on a + * `taskFlow` target, which keeps its flow's own cache policy. */ cache_ttl?: number; priority?: number; concurrency_limit?: number; diff --git a/system_prompts/auto-generated/skills/write-script-python3/SKILL.md b/system_prompts/auto-generated/skills/write-script-python3/SKILL.md index 8100eebc25..1ccb376a9f 100644 --- a/system_prompts/auto-generated/skills/write-script-python3/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-python3/SKILL.md @@ -857,6 +857,14 @@ def parse_sql_client_name(name: str) -> tuple[str, Optional[str]] # it grows with both the width of the fan-out and ``attempts``. Retries with # no ``delay`` all go out in a single round. # +# ``cache_ttl`` serves a previous result of the task for that many seconds +# instead of running it again. A task is keyed on its step key (its name and +# call order) and the workflow's input, not on the arguments it is called +# with, so cache one only when whether it runs, and what it receives, follow +# from the workflow's input alone. A ``task_script`` target is keyed on the +# arguments it is called with. It has no effect on a ``task_flow`` target, +# which keeps its flow's own cache policy. +# # Usage:: # # @task diff --git a/system_prompts/auto-generated/skills/write-workflow-as-code/SKILL.md b/system_prompts/auto-generated/skills/write-workflow-as-code/SKILL.md index c41bd54f04..8e5ab1f06c 100644 --- a/system_prompts/auto-generated/skills/write-workflow-as-code/SKILL.md +++ b/system_prompts/auto-generated/skills/write-workflow-as-code/SKILL.md @@ -277,6 +277,13 @@ export interface TaskRetry { export interface TaskOptions { timeout?: number; tag?: string; + /** Seconds during which a previous result of this task is served instead of + * running it again. A task written inline in the workflow is keyed on its + * step key (its name and call order) and the workflow's input, not on the + * arguments it is called with, so cache one only when whether it runs, and + * what it receives, follow from the workflow's input alone. A `taskScript` + * target is keyed on the arguments it is called with. It has no effect on a + * `taskFlow` target, which keeps its flow's own cache policy. */ cache_ttl?: number; priority?: number; concurrency_limit?: number; @@ -468,6 +475,14 @@ def get_resume_urls(approver: str = None, flow_level: bool = None) -> dict # it grows with both the width of the fan-out and ``attempts``. Retries with # no ``delay`` all go out in a single round. # +# ``cache_ttl`` serves a previous result of the task for that many seconds +# instead of running it again. A task is keyed on its step key (its name and +# call order) and the workflow's input, not on the arguments it is called +# with, so cache one only when whether it runs, and what it receives, follow +# from the workflow's input alone. A ``task_script`` target is keyed on the +# arguments it is called with. It has no effect on a ``task_flow`` target, +# which keeps its flow's own cache policy. +# # Usage:: # # @task diff --git a/typescript-client/client.ts b/typescript-client/client.ts index 7b10c732be..dd2123e746 100644 --- a/typescript-client/client.ts +++ b/typescript-client/client.ts @@ -1712,6 +1712,13 @@ export interface TaskRetry { export interface TaskOptions { timeout?: number; tag?: string; + /** Seconds during which a previous result of this task is served instead of + * running it again. A task written inline in the workflow is keyed on its + * step key (its name and call order) and the workflow's input, not on the + * arguments it is called with, so cache one only when whether it runs, and + * what it receives, follow from the workflow's input alone. A `taskScript` + * target is keyed on the arguments it is called with. It has no effect on a + * `taskFlow` target, which keeps its flow's own cache policy. */ cache_ttl?: number; priority?: number; concurrency_limit?: number; From b9b5988ebdf3edd75add445282977c516ef5ed51 Mon Sep 17 00:00:00 2001 From: Guilhem Date: Wed, 16 Sep 2026 10:42:47 +0200 Subject: [PATCH 37/44] fix: keep sidebar confirmation dialogs from being confined to the rail (#11158) Co-authored-by: Claude Fable 5.1 --- frontend/src/lib/components/home/ItemsList.svelte | 5 ++++- .../components/sidebar/DeleteForkedWorkspaceModal.svelte | 3 +++ frontend/src/lib/components/sidebar/SettingsMenu.svelte | 3 +++ frontend/src/lib/components/sidebar/SidebarContent.svelte | 3 +++ frontend/src/routes/(root)/(logged)/+layout.svelte | 6 +++++- 5 files changed, 18 insertions(+), 2 deletions(-) diff --git a/frontend/src/lib/components/home/ItemsList.svelte b/frontend/src/lib/components/home/ItemsList.svelte index 35b1c5a6f3..7038a316fa 100644 --- a/frontend/src/lib/components/home/ItemsList.svelte +++ b/frontend/src/lib/components/home/ItemsList.svelte @@ -2107,8 +2107,11 @@ } } + /* `backwards`, not `both`: the fill hides rows until their staggered start, but a forwards + fill would keep `transform` animated afterwards and confine the row's remove confirmation + (`position: fixed`) to the row. The `to` keyframe equals the row's resting style. */ .wm-imported > :global(*) { - animation: wm-row-in 260ms ease-out both; + animation: wm-row-in 260ms ease-out backwards; animation-delay: 320ms; } .wm-imported > :global(*:nth-child(1)) { diff --git a/frontend/src/lib/components/sidebar/DeleteForkedWorkspaceModal.svelte b/frontend/src/lib/components/sidebar/DeleteForkedWorkspaceModal.svelte index ff049ceddb..3a73758146 100644 --- a/frontend/src/lib/components/sidebar/DeleteForkedWorkspaceModal.svelte +++ b/frontend/src/lib/components/sidebar/DeleteForkedWorkspaceModal.svelte @@ -151,8 +151,11 @@ {#if currentWsIsFork} + { diff --git a/frontend/src/lib/components/sidebar/SettingsMenu.svelte b/frontend/src/lib/components/sidebar/SettingsMenu.svelte index 857e422bdd..e0951f37cd 100644 --- a/frontend/src/lib/components/sidebar/SettingsMenu.svelte +++ b/frontend/src/lib/components/sidebar/SettingsMenu.svelte @@ -396,8 +396,11 @@ {/snippet} + { diff --git a/frontend/src/lib/components/sidebar/SidebarContent.svelte b/frontend/src/lib/components/sidebar/SidebarContent.svelte index 7fa161dd2d..b43bc2988e 100644 --- a/frontend/src/lib/components/sidebar/SidebarContent.svelte +++ b/frontend/src/lib/components/sidebar/SidebarContent.svelte @@ -737,8 +737,11 @@
    + { diff --git a/frontend/src/routes/(root)/(logged)/+layout.svelte b/frontend/src/routes/(root)/(logged)/+layout.svelte index 225b9443d3..c82e8b9d54 100644 --- a/frontend/src/routes/(root)/(logged)/+layout.svelte +++ b/frontend/src/routes/(root)/(logged)/+layout.svelte @@ -1515,8 +1515,12 @@ } } + /* No forwards fill: a filled animation keeps `transform` animated after it ends, which makes + the rail the containing block for every `position: fixed` descendant — the confirmation + dialogs opened from the settings menu would be confined to the rail's column. The `to` + keyframe equals the rail's resting style, so nothing changes visually when the fill drops. */ :global(#sidebar.wm-sidebar-in) { - animation: wm-sidebar-in 500ms ease-out both; + animation: wm-sidebar-in 500ms ease-out; } @media (prefers-reduced-motion: reduce) { From 57a99f66a88f195cac8f583b59d69d627cc1ec1d Mon Sep 17 00:00:00 2001 From: hugocasa Date: Wed, 16 Sep 2026 10:45:01 +0200 Subject: [PATCH 38/44] feat: rename saved agents from the agent editor and flag broken links (#11147) * feat: list flows that link a saved agent and flag broken agent links * feat: rename saved agents from the agent editor and repoint the flow * fix: show an unreadable linked agent as not accessible, not missing * fix: address review nits on agent rename and missing-agent state * fix: open content search above modals and keep Escape for it * fix: register content search on the opener's overlay stack * docs: scope the global search z-index comment to the bases it clears * refactor: show linked agents' rename warning as for scripts and flows * fix: keep the failed-lookup rename warning to resources --- ...2e38967680a77316ee9a53e67d70d7df9f63.json} | 4 +- ...1892fa3899934e7e0e51ad23bfcd2d6d3d08.json} | 4 +- ...4250441863ab47c345d9685f53c7e224b4887.json | 15 +++ ...c6ac927bd0e6b0f383626e673225065ee90bf.json | 23 ++++ ...f3bee9b93ba0387f566ad7f502cbee818296e.json | 15 --- ...eea76ab0226a514d25712a733e01add508b71.json | 16 +++ ...d_agents_as_runnable_dependencies.down.sql | 9 ++ ...ked_agents_as_runnable_dependencies.up.sql | 21 +++ backend/summarized_schema.txt | 2 +- backend/windmill-api-flows/src/flows.rs | 32 ++++- .../windmill-api-workspaces/src/workspaces.rs | 4 +- backend/windmill-api/openapi.yaml | 19 +++ .../windmill-worker/src/worker_lockfiles.rs | 11 ++ .../src/lib/components/FlowBuilder.svelte | 4 +- frontend/src/lib/components/Path.svelte | 69 ++++++---- .../lib/components/flows/FlowEditor.svelte | 2 + .../lib/components/flows/agentDraft.svelte.ts | 52 +++++--- .../flows/content/AgentEditorHost.svelte | 40 +++++- .../flows/content/AgentEditorModal.svelte | 28 +++- .../flows/content/AgentResourceBar.svelte | 125 +++++++++++++----- .../flows/linkedAgentDrafts.test.ts | 33 +++++ .../lib/components/flows/linkedAgentDrafts.ts | 21 ++- .../search/GlobalSearchModal.svelte | 31 ++++- frontend/src/lib/zIndexes.ts | 4 + .../src/routes/(root)/(logged)/+layout.svelte | 7 +- .../(root)/(logged)/resources/+page.svelte | 14 +- 26 files changed, 482 insertions(+), 123 deletions(-) rename backend/.sqlx/{query-00c0ae12b19ba495f307f0ce6b4833947c5b3fe45826fc5468e326d171d95236.json => query-0c49b098051900b834cb791e37af2e38967680a77316ee9a53e67d70d7df9f63.json} (78%) rename backend/.sqlx/{query-dc5eeb7b7bf0b7217ef66eb950ab7e9cf578bba7bd1eec981526be4067bcb314.json => query-139e153d1ebe584e878d3b2569551892fa3899934e7e0e51ad23bfcd2d6d3d08.json} (76%) create mode 100644 backend/.sqlx/query-2aa87574b437f0e29991696564c4250441863ab47c345d9685f53c7e224b4887.json create mode 100644 backend/.sqlx/query-46e65196c2a4f07d171a22f1e45c6ac927bd0e6b0f383626e673225065ee90bf.json delete mode 100644 backend/.sqlx/query-a54e2334c365f90577f68ebefc8f3bee9b93ba0387f566ad7f502cbee818296e.json create mode 100644 backend/.sqlx/query-fa8c36eda6d4cb64b4ac5979cc4eea76ab0226a514d25712a733e01add508b71.json create mode 100644 backend/migrations/20260915111928_record_linked_agents_as_runnable_dependencies.down.sql create mode 100644 backend/migrations/20260915111928_record_linked_agents_as_runnable_dependencies.up.sql diff --git a/backend/.sqlx/query-00c0ae12b19ba495f307f0ce6b4833947c5b3fe45826fc5468e326d171d95236.json b/backend/.sqlx/query-0c49b098051900b834cb791e37af2e38967680a77316ee9a53e67d70d7df9f63.json similarity index 78% rename from backend/.sqlx/query-00c0ae12b19ba495f307f0ce6b4833947c5b3fe45826fc5468e326d171d95236.json rename to backend/.sqlx/query-0c49b098051900b834cb791e37af2e38967680a77316ee9a53e67d70d7df9f63.json index f950c5bf57..da25d2fcf8 100644 --- a/backend/.sqlx/query-00c0ae12b19ba495f307f0ce6b4833947c5b3fe45826fc5468e326d171d95236.json +++ b/backend/.sqlx/query-0c49b098051900b834cb791e37af2e38967680a77316ee9a53e67d70d7df9f63.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT f.path\n FROM workspace_runnable_dependencies wru \n JOIN flow f\n ON wru.flow_path = f.path AND wru.workspace_id = f.workspace_id\n WHERE wru.runnable_path = $1 AND wru.runnable_is_flow = $2 AND wru.workspace_id = $3", + "query": "SELECT f.path\n FROM workspace_runnable_dependencies wru \n JOIN flow f\n ON wru.flow_path = f.path AND wru.workspace_id = f.workspace_id\n WHERE wru.runnable_path = $1 AND wru.runnable_is_flow = $2 AND NOT wru.runnable_is_agent AND wru.workspace_id = $3", "describe": { "columns": [ { @@ -20,5 +20,5 @@ false ] }, - "hash": "00c0ae12b19ba495f307f0ce6b4833947c5b3fe45826fc5468e326d171d95236" + "hash": "0c49b098051900b834cb791e37af2e38967680a77316ee9a53e67d70d7df9f63" } diff --git a/backend/.sqlx/query-dc5eeb7b7bf0b7217ef66eb950ab7e9cf578bba7bd1eec981526be4067bcb314.json b/backend/.sqlx/query-139e153d1ebe584e878d3b2569551892fa3899934e7e0e51ad23bfcd2d6d3d08.json similarity index 76% rename from backend/.sqlx/query-dc5eeb7b7bf0b7217ef66eb950ab7e9cf578bba7bd1eec981526be4067bcb314.json rename to backend/.sqlx/query-139e153d1ebe584e878d3b2569551892fa3899934e7e0e51ad23bfcd2d6d3d08.json index 37115e4ab7..780fb75ea9 100644 --- a/backend/.sqlx/query-dc5eeb7b7bf0b7217ef66eb950ab7e9cf578bba7bd1eec981526be4067bcb314.json +++ b/backend/.sqlx/query-139e153d1ebe584e878d3b2569551892fa3899934e7e0e51ad23bfcd2d6d3d08.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT DISTINCT f.path\n FROM workspace_runnable_dependencies wru \n JOIN flow f\n ON wru.flow_path = f.path AND wru.workspace_id = f.workspace_id\n WHERE wru.runnable_path LIKE $1 || '%' AND wru.runnable_is_flow = $2 AND wru.workspace_id = $3", + "query": "SELECT DISTINCT f.path\n FROM workspace_runnable_dependencies wru \n JOIN flow f\n ON wru.flow_path = f.path AND wru.workspace_id = f.workspace_id\n WHERE wru.runnable_path LIKE $1 || '%' AND wru.runnable_is_flow = $2 AND NOT wru.runnable_is_agent AND wru.workspace_id = $3", "describe": { "columns": [ { @@ -20,5 +20,5 @@ false ] }, - "hash": "dc5eeb7b7bf0b7217ef66eb950ab7e9cf578bba7bd1eec981526be4067bcb314" + "hash": "139e153d1ebe584e878d3b2569551892fa3899934e7e0e51ad23bfcd2d6d3d08" } diff --git a/backend/.sqlx/query-2aa87574b437f0e29991696564c4250441863ab47c345d9685f53c7e224b4887.json b/backend/.sqlx/query-2aa87574b437f0e29991696564c4250441863ab47c345d9685f53c7e224b4887.json new file mode 100644 index 0000000000..919d930f70 --- /dev/null +++ b/backend/.sqlx/query-2aa87574b437f0e29991696564c4250441863ab47c345d9685f53c7e224b4887.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace_runnable_dependencies (flow_path, runnable_path, script_hash, runnable_is_flow, runnable_is_agent, workspace_id, app_path)\n SELECT flow_path, runnable_path, script_hash, runnable_is_flow, runnable_is_agent, $1, app_path\n FROM workspace_runnable_dependencies\n WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "2aa87574b437f0e29991696564c4250441863ab47c345d9685f53c7e224b4887" +} diff --git a/backend/.sqlx/query-46e65196c2a4f07d171a22f1e45c6ac927bd0e6b0f383626e673225065ee90bf.json b/backend/.sqlx/query-46e65196c2a4f07d171a22f1e45c6ac927bd0e6b0f383626e673225065ee90bf.json new file mode 100644 index 0000000000..9b361b8f53 --- /dev/null +++ b/backend/.sqlx/query-46e65196c2a4f07d171a22f1e45c6ac927bd0e6b0f383626e673225065ee90bf.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT DISTINCT f.path\n FROM workspace_runnable_dependencies wru\n JOIN flow f\n ON wru.flow_path = f.path AND wru.workspace_id = f.workspace_id\n WHERE wru.runnable_path = $1 AND wru.runnable_is_agent AND wru.workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "46e65196c2a4f07d171a22f1e45c6ac927bd0e6b0f383626e673225065ee90bf" +} diff --git a/backend/.sqlx/query-a54e2334c365f90577f68ebefc8f3bee9b93ba0387f566ad7f502cbee818296e.json b/backend/.sqlx/query-a54e2334c365f90577f68ebefc8f3bee9b93ba0387f566ad7f502cbee818296e.json deleted file mode 100644 index 4e3b39403f..0000000000 --- a/backend/.sqlx/query-a54e2334c365f90577f68ebefc8f3bee9b93ba0387f566ad7f502cbee818296e.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO workspace_runnable_dependencies (flow_path, runnable_path, script_hash, runnable_is_flow, workspace_id, app_path)\n SELECT flow_path, runnable_path, script_hash, runnable_is_flow, $1, app_path\n FROM workspace_runnable_dependencies\n WHERE workspace_id = $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Text" - ] - }, - "nullable": [] - }, - "hash": "a54e2334c365f90577f68ebefc8f3bee9b93ba0387f566ad7f502cbee818296e" -} diff --git a/backend/.sqlx/query-fa8c36eda6d4cb64b4ac5979cc4eea76ab0226a514d25712a733e01add508b71.json b/backend/.sqlx/query-fa8c36eda6d4cb64b4ac5979cc4eea76ab0226a514d25712a733e01add508b71.json new file mode 100644 index 0000000000..6a4b3fe6de --- /dev/null +++ b/backend/.sqlx/query-fa8c36eda6d4cb64b4ac5979cc4eea76ab0226a514d25712a733e01add508b71.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace_runnable_dependencies (flow_path, runnable_path, runnable_is_flow, runnable_is_agent, workspace_id) VALUES ($1, $2, FALSE, TRUE, $3) ON CONFLICT DO NOTHING", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "fa8c36eda6d4cb64b4ac5979cc4eea76ab0226a514d25712a733e01add508b71" +} diff --git a/backend/migrations/20260915111928_record_linked_agents_as_runnable_dependencies.down.sql b/backend/migrations/20260915111928_record_linked_agents_as_runnable_dependencies.down.sql new file mode 100644 index 0000000000..c749fbe1b7 --- /dev/null +++ b/backend/migrations/20260915111928_record_linked_agents_as_runnable_dependencies.down.sql @@ -0,0 +1,9 @@ +DELETE FROM workspace_runnable_dependencies WHERE runnable_is_agent; + +DROP INDEX flow_workspace_without_hash_unique_idx; + +CREATE UNIQUE INDEX flow_workspace_without_hash_unique_idx + ON workspace_runnable_dependencies (flow_path, runnable_path, runnable_is_flow, workspace_id) + WHERE script_hash IS NULL; + +ALTER TABLE workspace_runnable_dependencies DROP COLUMN runnable_is_agent; diff --git a/backend/migrations/20260915111928_record_linked_agents_as_runnable_dependencies.up.sql b/backend/migrations/20260915111928_record_linked_agents_as_runnable_dependencies.up.sql new file mode 100644 index 0000000000..eab08e5c17 --- /dev/null +++ b/backend/migrations/20260915111928_record_linked_agents_as_runnable_dependencies.up.sql @@ -0,0 +1,21 @@ +-- A flow step linked to a saved agent (an `ai_agent` resource) is recorded next to the scripts and +-- subflows the flow runs, so renaming the agent can name the flows it would break. An agent row is +-- neither a script nor a flow: readers of script usages have to exclude it. +ALTER TABLE workspace_runnable_dependencies + ADD COLUMN runnable_is_agent BOOLEAN NOT NULL DEFAULT false; + +-- A script step and a linked agent can share a path. Without the flag in the key, the second +-- insert's ON CONFLICT DO NOTHING would silently drop one of the two rows. +DROP INDEX flow_workspace_without_hash_unique_idx; + +CREATE UNIQUE INDEX flow_workspace_without_hash_unique_idx + ON workspace_runnable_dependencies (flow_path, runnable_path, runnable_is_flow, runnable_is_agent, workspace_id) + WHERE script_hash IS NULL; + +-- The worker only records a flow when it is next deployed, so seed the ones already linking an +-- agent from their current value. +INSERT INTO workspace_runnable_dependencies (flow_path, runnable_path, runnable_is_flow, runnable_is_agent, workspace_id) +SELECT DISTINCT f.path, agent_ref #>> '{}', false, true, f.workspace_id +FROM flow f +CROSS JOIN LATERAL jsonb_path_query(f.value, 'lax $.** ? (@.type == "aiagent" && @.agent.type() == "string").agent') AS agent_ref +ON CONFLICT DO NOTHING; diff --git a/backend/summarized_schema.txt b/backend/summarized_schema.txt index b93a4bf6fc..1e27ee1ce4 100644 --- a/backend/summarized_schema.txt +++ b/backend/summarized_schema.txt @@ -232,7 +232,7 @@ workspace_key: workspace_id(char), kind(workspace_key_kind), key(char) FK: (workspace_id) -> workspace(id) workspace_protection_rule: workspace_id(char), name(char), rules(int), bypass_groups(text[]), bypass_users(text[]), created_at(ts) FK: (workspace_id) -> workspace(id) -workspace_runnable_dependencies: flow_path(char), runnable_path(char), script_hash(bigint), runnable_is_flow(bool), workspace_id(char), app_path(char), id(bigint) +workspace_runnable_dependencies: flow_path(char), runnable_path(char), script_hash(bigint), runnable_is_flow(bool), workspace_id(char), app_path(char), id(bigint), runnable_is_agent(bool) FK: (app_path, workspace_id) -> app(path, workspace_id) | (flow_path, workspace_id) -> flow(path, workspace_id) workspace_settings: workspace_id(char), slack_team_id(char), slack_name(char), slack_command_script(char), slack_email(char), customer_id(char), plan(char), webhook(text), ai_config(jsonb), large_file_storage(jsonb), git_sync(jsonb), default_app(char), default_scripts(jsonb), deploy_ui(jsonb), mute_critical_alerts(bool), color(char), operator_settings(jsonb), teams_command_script(text), teams_team_id(text), teams_team_name(text), git_app_installations(jsonb), ducklake(jsonb), slack_oauth_client_id(char), slack_oauth_client_secret(char), datatable(jsonb), teams_team_guid(text), auto_invite(jsonb), error_handler(jsonb), success_handler(jsonb), public_app_execution_limit_per_minute(int), dbt_warehouses(jsonb), guest_access_enabled(bool), guest_jwt_public_key(text), guest_jwt_jwks_url(text), ai_sessions_backup_generation(int) FK: (workspace_id) -> workspace(id) diff --git a/backend/windmill-api-flows/src/flows.rs b/backend/windmill-api-flows/src/flows.rs index 2093ac5fec..525e3cca46 100644 --- a/backend/windmill-api-flows/src/flows.rs +++ b/backend/windmill-api-flows/src/flows.rs @@ -77,6 +77,10 @@ pub fn workspaced_service() -> Router { "/list_paths_from_workspace_runnable/{runnable_kind}/{*path}", get(list_paths_from_workspace_runnable), ) + .route( + "/list_paths_linking_agent/{*path}", + get(list_paths_linking_agent), + ) .route("/history_update/v/{version}", post(update_flow_history)) .route("/get/v/{version}", get(get_flow_version_by_id)) .route("/get/v/{version}/p/{*path}", get(get_flow_version)) @@ -508,7 +512,7 @@ async fn list_paths_from_workspace_runnable( FROM workspace_runnable_dependencies wru JOIN flow f ON wru.flow_path = f.path AND wru.workspace_id = f.workspace_id - WHERE wru.runnable_path LIKE $1 || '%' AND wru.runnable_is_flow = $2 AND wru.workspace_id = $3"#, + WHERE wru.runnable_path LIKE $1 || '%' AND wru.runnable_is_flow = $2 AND NOT wru.runnable_is_agent AND wru.workspace_id = $3"#, path, matches!(runnable_kind, RunnableKind::Flow), w_id @@ -521,7 +525,7 @@ async fn list_paths_from_workspace_runnable( FROM workspace_runnable_dependencies wru JOIN flow f ON wru.flow_path = f.path AND wru.workspace_id = f.workspace_id - WHERE wru.runnable_path = $1 AND wru.runnable_is_flow = $2 AND wru.workspace_id = $3"#, + WHERE wru.runnable_path = $1 AND wru.runnable_is_flow = $2 AND NOT wru.runnable_is_agent AND wru.workspace_id = $3"#, path, matches!(runnable_kind, RunnableKind::Flow), w_id @@ -534,6 +538,30 @@ async fn list_paths_from_workspace_runnable( Ok(Json(runnables)) } +/// Flows with a step linked to the `ai_agent` resource at `path`, as of their last deploy. +async fn list_paths_linking_agent( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, +) -> JsonResult> { + let path = path.to_path(); + check_scopes(&authed, || format!("flows:read:agent/{}", path))?; + let mut tx = user_db.begin(&authed).await?; + let flows = sqlx::query_scalar!( + r#"SELECT DISTINCT f.path + FROM workspace_runnable_dependencies wru + JOIN flow f + ON wru.flow_path = f.path AND wru.workspace_id = f.workspace_id + WHERE wru.runnable_path = $1 AND wru.runnable_is_agent AND wru.workspace_id = $2"#, + path, + w_id + ) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + Ok(Json(flows)) +} + async fn validate_flow(new_flow: &NewFlow) -> error::Result<()> { #[cfg(not(feature = "enterprise"))] if new_flow.ws_error_handler_muted.is_some_and(|val| val) { diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 3330ed697f..3e48e7baca 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -7471,8 +7471,8 @@ async fn clone_workspace_runnable_dependencies( ) -> Result<()> { // Clone workspace_runnable_dependencies sqlx::query!( - "INSERT INTO workspace_runnable_dependencies (flow_path, runnable_path, script_hash, runnable_is_flow, workspace_id, app_path) - SELECT flow_path, runnable_path, script_hash, runnable_is_flow, $1, app_path + "INSERT INTO workspace_runnable_dependencies (flow_path, runnable_path, script_hash, runnable_is_flow, runnable_is_agent, workspace_id, app_path) + SELECT flow_path, runnable_path, script_hash, runnable_is_flow, runnable_is_agent, $1, app_path FROM workspace_runnable_dependencies WHERE workspace_id = $2", target_workspace_id, diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index c88a298646..8f188bf763 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -12103,6 +12103,25 @@ paths: items: type: string + /w/{workspace}/flows/list_paths_linking_agent/{path}: + get: + summary: list flow paths with a step linked to a saved agent + operationId: listFlowPathsLinkingAgent + tags: + - flow + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Path" + responses: + "200": + description: paths of the flows linking the `ai_agent` resource, as of their last deploy + content: + application/json: + schema: + type: array + items: + type: string + /w/{workspace}/flows/get/v/{version}: get: summary: get flow version diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index 1f3b9f1d56..03150beb00 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -1795,6 +1795,17 @@ async fn lock_modules( agent, tool_inputs, } => { + if let Some(agent_path) = agent.as_deref().filter(|_| !skip_flow_update) { + sqlx::query!( + "INSERT INTO workspace_runnable_dependencies (flow_path, runnable_path, runnable_is_flow, runnable_is_agent, workspace_id) VALUES ($1, $2, FALSE, TRUE, $3) ON CONFLICT DO NOTHING", + job_path, + agent_path, + job.workspace_id, + ) + .execute(db) + .await?; + } + // Extract FlowModules from tools and track their original indices // MCP tools don't need locking, so we filter them out let mut flow_modules = Vec::new(); diff --git a/frontend/src/lib/components/FlowBuilder.svelte b/frontend/src/lib/components/FlowBuilder.svelte index 6888c61f2d..97b871921c 100644 --- a/frontend/src/lib/components/FlowBuilder.svelte +++ b/frontend/src/lib/components/FlowBuilder.svelte @@ -510,8 +510,8 @@ draftAgents.map((a) => [a.path, agentDraftCanWrite(a, user ?? $userStore ?? undefined)]) ) // The path is passed, so a draft that renames the agent is refused here too: a rename is - // the resource editor's to deploy, and this dialog lists the agent under the path the - // flow links. + // the agent editor's to deploy, and this dialog lists the agent under the path the flow + // links. agentRefusal = Object.fromEntries( draftAgents.map((a) => [a.path, agentDraftDeployRefusal(a.state, a.path)]) ) diff --git a/frontend/src/lib/components/Path.svelte b/frontend/src/lib/components/Path.svelte index b45d980246..ee87cc9511 100644 --- a/frontend/src/lib/components/Path.svelte +++ b/frontend/src/lib/components/Path.svelte @@ -30,6 +30,7 @@ import { createEventDispatcher, getContext, untrack } from 'svelte' import { writable } from 'svelte/store' import { Alert, Button } from './common' + import { overlayStack, type OverlayStack } from './common/overlayHost.svelte' import { random_adj } from './random_positive_adjetive' import { ChevronDown, Copy, SearchCode } from 'lucide-svelte' import Tooltip from './Tooltip.svelte' @@ -405,9 +406,11 @@ } }) - const openSearchWithPrefilledText: (t?: string) => void = getContext( + const openSearchWithPrefilledText: (t?: string, stack?: OverlayStack) => void = getContext( 'openSearchWithPrefilledText' ) + // Handed to the search so it stacks above the modal or drawer this field sits in. + const searchStack = overlayStack() $effect.pre(() => { ;[meta?.name, meta?.owner, meta?.ownerKind] @@ -451,14 +454,18 @@ initialPath !== path ) let pathUsageInFlowsPromise = $derived( - (kind == 'script' || kind == 'flow') && - ws && - initialPath && - FlowService.listFlowPathsFromWorkspaceRunnable({ - workspace: ws, - path: initialPath, - runnableKind: kind - }) + ws && initialPath + ? kind == 'script' || kind == 'flow' + ? FlowService.listFlowPathsFromWorkspaceRunnable({ + workspace: ws, + path: initialPath, + runnableKind: kind + }) + : kind == 'resource' + ? // Only steps linked to a saved agent are tracked; other `$res:` references are not. + FlowService.listFlowPathsLinkingAgent({ workspace: ws, path: initialPath }) + : undefined + : undefined ) let pathUsageInAppsPromise = $derived( (kind == 'script' || kind == 'flow') && @@ -647,24 +654,36 @@ {/if} + {:else if displayPathChangedWarning && kind == 'resource'} + {@render renameMayBreakWarning()} + {/if} + {:catch} + + {#if displayPathChangedWarning && kind == 'resource'} + {@render renameMayBreakWarning()} {/if} {/await} {:else if displayPathChangedWarning} - - You are renaming an item that may be depended upon by other items. This may break apps, flows - or resources. Find if it used elsewhere using the content search. Note that linked variables - and resources (having the same path) are automatically moved together. -
    - -
    -     + {@render renameMayBreakWarning()} {/if}
    + +{#snippet renameMayBreakWarning()} + + You are renaming an item that may be depended upon by other items. This may break apps, flows or + resources. Find if it used elsewhere using the content search. Note that linked variables and + resources (having the same path) are automatically moved together. +
    + +
    +     +{/snippet} diff --git a/frontend/src/lib/components/flows/FlowEditor.svelte b/frontend/src/lib/components/flows/FlowEditor.svelte index 73482817ec..bf25a1dcda 100644 --- a/frontend/src/lib/components/flows/FlowEditor.svelte +++ b/frontend/src/lib/components/flows/FlowEditor.svelte @@ -4,6 +4,7 @@ import FlowEditorPanel from './content/FlowEditorPanel.svelte' import { agentEditorTarget, type AgentEditorTarget } from './agentEditorStore.svelte' import AgentEditorModal from './content/AgentEditorModal.svelte' + import { repointLinkedAgent } from './linkedAgentDrafts' import FlowModuleSchemaMap from './map/FlowModuleSchemaMap.svelte' import type { OpenInSessionSource } from '$lib/components/sessions/OpenInSessionButton.svelte' import WindmillIcon from '../icons/WindmillIcon.svelte' @@ -551,4 +552,5 @@ t.host?.flowPath === $pathStore && targetWorkspace(t) === editorWorkspace} + onRenamed={(from, to) => repointLinkedAgent(flowStore.val.value, from, to)} /> diff --git a/frontend/src/lib/components/flows/agentDraft.svelte.ts b/frontend/src/lib/components/flows/agentDraft.svelte.ts index d4f8fa35de..5ee65cd98a 100644 --- a/frontend/src/lib/components/flows/agentDraft.svelte.ts +++ b/frontend/src/lib/components/flows/agentDraft.svelte.ts @@ -112,12 +112,11 @@ export function agentDraftDeployRefusal( if (blocked) { return blocked } - // Renaming is not the agent editor's to do: moving the resource leaves every step that links to - // it naming a path that no longer exists, and reconciling those is a feature of its own. A - // renamed path can still reach here, the generic editor writing the same draft row and offering - // a path field, so refuse it rather than performing half of a rename. + // A rename is the agent editor's to deploy: it repoints the steps of the flow it was opened from. + // Deployed from anywhere that names the path it writes to (a flow's deploy dialog, which lists the + // agent under the path the flow links), it would move the agent out from under that flow. if (currentPath && state.path !== currentPath) { - return `This draft renames the agent to ${state.path}. Deploy it from the resource editor instead.` + return `This draft renames the agent to ${state.path}. Deploy it from the agent editor instead.` } // Only a draft naming another type: the load refuses a resource that is not an agent, while a // draft the generic resource editor wrote names no type at all and inherits the loaded one. @@ -132,6 +131,9 @@ export function agentDraftDeployRefusal( * persisted draft row: the form stays editable while a deploy is in flight. Surfaces that deploy * the row itself go through `deployDraft` instead. * + * `fromPath` is the path the editor loaded; `state.path` differs from it when the draft renames the + * agent, and the update then moves the resource there. + * * `notAnAgent` separates the one failure that invalidates the caller's whole view of the path, its * holding something else now, from a write that merely failed. */ @@ -139,6 +141,7 @@ type AgentWriteResult = { ok: true } | { ok: false; error: string; notAnAgent?: async function writeAgentResource( workspace: string, + fromPath: string, state: AgentResourceState, noDeployed: boolean ): Promise { @@ -161,12 +164,12 @@ async function writeAgentResource( // its own: were the path deleted and recreated as something else meanwhile, this write // would put an agent config inside that resource. Reading it again narrows the window to // the request rather than to however long the editor or the dialog stayed open. - const current = await ResourceService.getResource({ workspace, path: state.path }) - const refused = agentEditorRefusal(state.path, current.resource_type) + const current = await ResourceService.getResource({ workspace, path: fromPath }) + const refused = agentEditorRefusal(fromPath, current.resource_type) if (refused) { return { ok: false, error: refused, notAnAgent: true } } - await ResourceService.updateResource({ workspace, path: state.path, requestBody: body }) + await ResourceService.updateResource({ workspace, path: fromPath, requestBody: body }) } } catch (err) { return { ok: false, error: `Could not save agent: ${err}` } @@ -193,8 +196,9 @@ export interface AgentDraftHandle { /** Why this path cannot be edited here, if it cannot. Render it instead of the form. */ readonly refusal: string | undefined readonly sync: TriggerDraftSync - /** Write the current state to the resource and drop the draft. */ - deploy: () => Promise + /** Write the current state to the resource and drop the draft. Resolves to the path written, + * which differs from the one loaded when the draft renames the agent, or undefined on failure. */ + deploy: () => Promise } /** @@ -333,21 +337,23 @@ export function useAgentDraft(opts: AgentDraftOptions): AgentDraftHandle { }) }) - async function deploy(): Promise { + async function deploy(): Promise { const ws = opts.workspace() + const fromPath = opts.path() const s = state - if (!ws || !s) return false - const refused = agentDraftDeployRefusal(s, opts.path()) + if (!ws || !fromPath || !s) return undefined + // No path to hold the draft to: renaming is this editor's to deploy. + const refused = agentDraftDeployRefusal(s, undefined) if (refused) { sendUserToast(refused, true) - return false + return undefined } // The form stays editable while the request is in flight, so everything below works from a // snapshot taken now. Adopting the live state as `deployed` afterwards would count an edit // made during the request as saved, and the banner would clear on a value the server never // received; against the snapshot it stays a draft, which is what it is. const submitted = structuredClone($state.snapshot(s)) as AgentResourceState - const written = await writeAgentResource(ws, submitted, noDeployed) + const written = await writeAgentResource(ws, fromPath, submitted, noDeployed) if (!written.ok) { // A path that is no longer an agent tears this editor down; anything else is a plain error // the user can retry from the form as it stands. @@ -356,29 +362,31 @@ export function useAgentDraft(opts: AgentDraftOptions): AgentDraftHandle { } else { sendUserToast(written.error, true) } - return false + return undefined } // The counter the step card's write-back used to report, from the surface that now owns the // write: a deploy here reaches every flow linking this agent. logReusableAgentUsage(noDeployed ? 'saved' : 'updated') deployed = submitted noDeployed = false + const renamed = submitted.path !== fromPath // Only when the form still holds exactly what was sent. `discard` resets the handle's cell to // what it is given, and the apply-effect copies that back over the form: against an edit made // while the request was in flight that would erase it, draft and all. Such an edit is a real - // unsaved change over the version just deployed, so it keeps its draft and its banner. - if (!deepEqual($state.snapshot(state), submitted)) { + // unsaved change over the version just deployed, so it keeps its draft and its banner. Not + // after a rename: the draft is keyed on a path that no longer names the agent. + if (!renamed && !deepEqual($state.snapshot(state), submitted)) { sendUserToast(`Saved agent ${submitted.path}. Later edits are still unsaved`) loadedFor = `${ws}:${submitted.path}` - return true + return submitted.path } // `discard`, not `remove`: it resets the handle's cell to what was just saved, so the // apply-effect cannot bounce the form back to the now-stale draft. - sync.discard(opts.path()!, submitted) + sync.discard(fromPath, submitted) // A rename moves the row, so the next load must not reuse the old key. loadedFor = `${ws}:${submitted.path}` - sendUserToast(`Saved agent ${submitted.path}`) - return true + sendUserToast(renamed ? `Renamed agent to ${submitted.path}` : `Saved agent ${submitted.path}`) + return submitted.path } return { diff --git a/frontend/src/lib/components/flows/content/AgentEditorHost.svelte b/frontend/src/lib/components/flows/content/AgentEditorHost.svelte index 9fd562692c..b6a98160be 100644 --- a/frontend/src/lib/components/flows/content/AgentEditorHost.svelte +++ b/frontend/src/lib/components/flows/content/AgentEditorHost.svelte @@ -33,6 +33,9 @@ import { AGENT_EDITOR_RUN_INPUTS, AGENT_TOOLS_ROW } from '../agentFormFields' import { toolDisplayName, type AgentTool } from '../agentToolUtils' import { useAgentDraft } from '../agentDraft.svelte' + import Path from '$lib/components/Path.svelte' + import Label from '$lib/components/Label.svelte' + import { sendUserToast } from '$lib/toast' interface Props { /** The `ai_agent` resource being edited. */ @@ -320,13 +323,19 @@ if (toolId === id) onSelectTool?.(undefined) } + /** The path field's own verdict (a taken path, an invalid name), which the server would otherwise + * only report after the request. */ + let pathError = $state('') + export function deploy(): Promise { - return draft.deploy().then(async (ok) => { - // The path this editor opened, not the draft's live one: `deploy` refuses a renaming draft, - // so the write always lands here, while the shared draft can be repointed by another tab - // mid-request and would send the reconciliation after a resource nobody wrote. - if (ok) await onSaved?.(path) - return ok + if (pathError) { + sendUserToast(`Cannot deploy the agent: ${pathError}`, true) + return Promise.resolve(false) + } + return draft.deploy().then(async (written) => { + // The path the write landed on, which a rename moves off the one this editor opened. + if (written) await onSaved?.(written) + return written !== undefined }) } export function draftHandle() { @@ -352,6 +361,25 @@
    +
    + +
    boolean + /** A deploy moved the agent from `from` to `to`. What names the old path belongs to the surface + * that opened the editor: a flow's own steps, a page's URL. */ + onRenamed?: (from: string, to: string) => void } - let { enableAi = false, owns }: Props = $props() + let { enableAi = false, owns, onRenamed = undefined }: Props = $props() // Every target names the surface that opened it, and only a flow step or a resource row can: // an agent used as a tool of the agent being edited stays part of it, with no way in this editor @@ -178,9 +182,10 @@ if (!at.host) return // The host graph resolves a linked agent's tool nodes from the resource, so it has to re-read // what the write just changed. Every step of that flow linking this agent, not only the one - // the editor was opened from: they all show tools the write may have moved. + // the editor was opened from: they all show tools the write may have moved. Looked up under + // the path the editor opened, since a rename is about to move those steps off it. const scope = linkedToolsScope(at.ws, at.host.flowPath) - const moduleIds = new Set(linkedModulesForAgent(scope, path)) + const moduleIds = new Set(linkedModulesForAgent(scope, at.path)) moduleIds.add(at.host.moduleId) return Promise.all( // With the draft: a deploy leaves none, but a version restore leaves the draft standing and @@ -202,10 +207,21 @@ } } - /** What a successful deploy has to reconcile. The path is the one it wrote, which `deploy` holds - * to the one the editor opened: this editor does not rename. */ + /** What a successful deploy has to reconcile. `savedPath` is the path it wrote, which a rename + * moves off the one the editor opened. */ async function onSaved(savedPath: string) { - await reconcile(deployingFor ?? currentWriteTarget(), savedPath) + const at = deployingFor ?? currentWriteTarget() + // Before the rename is announced: it finds the steps to refresh under the old path. + const reconciled = reconcile(at, savedPath) + if (at && savedPath !== at.path) { + onRenamed?.(at.path, savedPath) + // The dialog is keyed on the path, so this reloads it on the renamed agent. Only while it + // still shows the one deployed: it can be closed or pointed elsewhere mid-request. + if (target?.path === at.path) { + openAgentEditor({ path: savedPath, workspace: target.workspace, host: target.host }) + } + } + await reconciled } diff --git a/frontend/src/lib/components/flows/content/AgentResourceBar.svelte b/frontend/src/lib/components/flows/content/AgentResourceBar.svelte index 96548b10af..fafe69ebb4 100644 --- a/frontend/src/lib/components/flows/content/AgentResourceBar.svelte +++ b/frontend/src/lib/components/flows/content/AgentResourceBar.svelte @@ -33,7 +33,11 @@ } from '../linkedAgentToolsStore.svelte' import { logReusableAgentUsage } from '../agentTelemetry' import { claimLinkedToolsFetch } from '../flowState' - import { AgentDraftUnavailable, fetchAgentWithDraft } from '../linkedAgentDrafts' + import { + AgentDraftUnavailable, + fetchAgentWithDraft, + isExpectedLinkFailure + } from '../linkedAgentDrafts' import type { AgentResourceState } from '../agentDraft.svelte' import { getLocalDraftHint } from '$lib/localDraftHints.svelte' import Tooltip from '$lib/components/meltComponents/Tooltip.svelte' @@ -103,6 +107,30 @@ fromDraft: boolean providerPath?: string providerOk: boolean + /** The link cannot be read. `missing` (404): nothing exists at the path, the agent having been + * renamed or deleted. `forbidden` (401/403): it exists and this user is refused it, a folder + * they cannot read included, which says nothing about whether a run of the flow can read it. + * Returned rather than thrown so it is guarded like any result. */ + unavailable?: 'missing' | 'forbidden' + } + + async function fetchLinkedAgent( + path: string, + ws: string + ): Promise<{ response: Resource; draft: AgentResourceState | undefined }> { + try { + return await fetchAgentWithDraft(path, ws) + } catch (err) { + // Only the DRAFT was unreadable. This card is a display, so fall back to the deployed + // agent rather than rendering one with no brain and no tools, which reads as "the agent + // is empty" while the Draft badge still says it has unsaved changes. Same fallback the + // graph's tool nodes take; the paths that run or deploy the draft still refuse. + if (!(err instanceof AgentDraftUnavailable)) throw err + return { + response: await ResourceService.getResource({ workspace: ws, path }), + draft: undefined + } + } } // A linked agent is rigid and read-only: its brain and tools come from the resource. We @@ -112,29 +140,27 @@ let linkedResource = resource( () => ({ ws, path: agent, writes, draftSaves }), async ({ ws, path, writes, draftSaves }): Promise => { + const empty = { + ws, + path, + writes, + draftSaves, + config: {}, + tools: [], + fromDraft: false, + providerOk: true + } if (!ws || !path) { - return { - ws, - path, - writes, - draftSaves, - config: {}, - tools: [], - fromDraft: false, - providerOk: true - } + return empty } let response: Resource let draft: AgentResourceState | undefined try { - ;({ response, draft } = await fetchAgentWithDraft(path, ws)) + ;({ response, draft } = await fetchLinkedAgent(path, ws)) } catch (err) { - // Only the DRAFT was unreadable. This card is a display, so fall back to the deployed - // agent rather than rendering one with no brain and no tools, which reads as "the agent - // is empty" while the Draft badge still says it has unsaved changes. Same fallback the - // graph's tool nodes take; the paths that run or deploy the draft still refuse. - if (!(err instanceof AgentDraftUnavailable)) throw err - response = await ResourceService.getResource({ workspace: ws, path }) + if (!isExpectedLinkFailure(err)) throw err + const status = (err as { status?: number }).status + return { ...empty, unavailable: status === 404 ? 'missing' : 'forbidden' } } const cfg = (draft?.args ?? response.value ?? {}) as AIAgentConfig & { provider?: { resource?: string } @@ -189,6 +215,7 @@ let brainParams = $derived(summarizeAgentBrain(linkedInfo?.config)) let providerPath = $derived(linkedInfo?.providerPath) let providerOk = $derived(linkedInfo?.providerOk ?? true) + let unavailable = $derived(linkedInfo?.unavailable ?? false) // The hint flips on the first keystroke in the agent editor, so the badge does not wait for the // debounced autosave and the refetch behind it; the fetched answer covers a draft written // elsewhere, which no editor here has published an opinion about. @@ -468,6 +495,15 @@ } } + // A link naming nothing readable has nothing to fork. Dropping it leaves a standalone step with its + // flow-local inputs, to configure here or replace with a saved agent; the tool overrides were + // keyed by the missing agent's tools, so they go with it. + function removeLink() { + toolInputs = {} + agent = undefined + sendUserToast('Removed the link to the missing agent') + } + // Edit the saved agent itself. The step stays linked throughout: the edits live in the agent's // own resource draft, not in this step, so they survive leaving the flow and are the same edits // whichever flow — or the resources page — opened them. @@ -534,7 +570,7 @@ {/if} {/if} - {#if !fromAgentEditor} + {#if !fromAgentEditor && !unavailable}
    {#if showDetail && (brainParams.length > 0 || inheritedTools.length > 0)} @@ -581,7 +619,32 @@ {/if}
    - {#if !providerOk} + {#if unavailable === 'forbidden'} +
    + + You don't have access to {agent}, so its configuration + can't be shown or edited here. + +
    + {:else if unavailable === 'missing'} +
    + + No saved agent exists at {agent}. It may have been + renamed or deleted. Remove the link to configure the step here, or add the agent again + from Saved agents. +
    + +
    +     +
    + {:else if !providerOk}
    This agent's model provider{#if providerPath} diff --git a/frontend/src/lib/components/flows/linkedAgentDrafts.test.ts b/frontend/src/lib/components/flows/linkedAgentDrafts.test.ts index 563007615f..fa72aaba5c 100644 --- a/frontend/src/lib/components/flows/linkedAgentDrafts.test.ts +++ b/frontend/src/lib/components/flows/linkedAgentDrafts.test.ts @@ -4,6 +4,7 @@ import { inlineAgentDraft, inlineAgentDrafts, loadLinkedAgentDrafts, + repointLinkedAgent, type LinkedAgentDraft } from './linkedAgentDrafts' import { ResourceService, type FlowModule, type FlowValue } from '$lib/gen' @@ -118,6 +119,38 @@ describe('inlineAgentDrafts', () => { }) }) +describe('repointLinkedAgent', () => { + // A rename from the agent editor moves every step of the host flow onto the new path, nested ones + // included. Miss one and it silently stays linked to a path that no longer exists. + it('repoints linked steps at any depth and leaves other agents alone', () => { + const value = { + modules: [ + { id: 'a', value: { type: 'aiagent', agent: 'f/team/support', tools: [] } }, + { + id: 'b', + value: { + type: 'branchall', + branches: [ + { + modules: [ + { id: 'c', value: { type: 'aiagent', agent: 'f/team/support', tools: [] } }, + { id: 'd', value: { type: 'aiagent', agent: 'f/team/other', tools: [] } } + ] + } + ] + } + } + ] + } as unknown as FlowValue + + expect(repointLinkedAgent(value, 'f/team/support', 'f/team/helpdesk')).toEqual(['a', 'c']) + const branch = (value.modules[1].value as any).branches[0].modules + expect((value.modules[0].value as any).agent).toBe('f/team/helpdesk') + expect(branch[0].value.agent).toBe('f/team/helpdesk') + expect(branch[1].value.agent).toBe('f/team/other') + }) +}) + // A link the user cannot resolve is an ordinary state and must not block the flow; anything else is // an outage, and answering "no draft" to one would silently test or deploy against the deployed // agent while the editor shows the draft. diff --git a/frontend/src/lib/components/flows/linkedAgentDrafts.ts b/frontend/src/lib/components/flows/linkedAgentDrafts.ts index 8da732e7ff..65624d4429 100644 --- a/frontend/src/lib/components/flows/linkedAgentDrafts.ts +++ b/frontend/src/lib/components/flows/linkedAgentDrafts.ts @@ -35,6 +35,25 @@ export function linkedAgentPaths(value: FlowValue | undefined): string[] { return [...paths] } +/** Point every step of this flow linked to `from` at `to`, for an agent renamed from inside it. + * Returns the ids of the steps it moved. */ +export function repointLinkedAgent( + value: FlowValue | undefined, + from: string, + to: string +): string[] { + if (!value?.modules) return [] + const moved: string[] = [] + for (const module of dfs(value.modules, (m) => m)) { + const v = module?.value as { type?: string; agent?: string } | undefined + if (v?.type === 'aiagent' && v.agent === from) { + v.agent = to + moved.push(module.id) + } + } + return moved +} + /** * The unsaved draft for an agent, freshest first: the cell an open agent editor is writing, then * what a `get_draft` response carried. @@ -114,7 +133,7 @@ export function agentDraftCanWrite(draft: LinkedAgentDraft, user: UserExt | unde * neither should stop the caller — the flow still tests and deploys, against the deployed agent. * Every other failure is an outage, and answering "no draft" to one would quietly run or deploy * the wrong configuration, which is the whole thing this module exists to prevent. */ -function isExpectedLinkFailure(err: unknown): boolean { +export function isExpectedLinkFailure(err: unknown): boolean { const status = (err as { status?: number } | null | undefined)?.status return status === 401 || status === 403 || status === 404 } diff --git a/frontend/src/lib/components/search/GlobalSearchModal.svelte b/frontend/src/lib/components/search/GlobalSearchModal.svelte index 084ad9d492..b94e0faefa 100644 --- a/frontend/src/lib/components/search/GlobalSearchModal.svelte +++ b/frontend/src/lib/components/search/GlobalSearchModal.svelte @@ -29,6 +29,8 @@ WandSparkles } from 'lucide-svelte' import Portal from '$lib/components/Portal.svelte' + import { zIndexes } from '$lib/zIndexes' + import { overlayStack } from '$lib/components/common/overlayHost.svelte' import { twMerge } from 'tailwind-merge' import ContentSearchInner from '../ContentSearchInner.svelte' @@ -367,6 +369,7 @@ async function handleKeydown(event: KeyboardEvent) { if ((!isMac() ? event.ctrlKey : event.metaKey) && event.key === 'k') { event.preventDefault() + if (!open) openedOn = undefined await openModal() } if (open) { @@ -450,6 +453,24 @@ mouseMoved = true } + // On the overlay stack while open: a modal or drawer it was opened from arbitrates Escape on that + // stack, and would otherwise close itself on the key meant for the search above it. The opener + // names its stack, because a pane hosting an editor (a sessions tab) keeps its own. + const globalStack = overlayStack() + let openedOn: import('$lib/components/common/overlayHost.svelte').OverlayStack | undefined = + $state(undefined) + const STACK_ID = 'global-search' + $effect(() => { + if (!open) return + const stack = openedOn ?? globalStack + untrack(() => stack.val.push(STACK_ID)) + return () => { + untrack(() => { + stack.val = stack.val.filter((id) => id !== STACK_ID) + }) + } + }) + onMount(() => { window.addEventListener('keydown', handleKeydown) window.addEventListener('mousemove', handleMouseMove) @@ -559,7 +580,11 @@ } } - export async function openSearchWithPrefilledText(text?: string) { + export async function openSearchWithPrefilledText( + text?: string, + stack?: import('$lib/components/common/overlayHost.svelte').OverlayStack + ) { + openedOn = stack await openModal() searchTerm = text ?? searchTerm await handleSearch() @@ -618,9 +643,9 @@
    {#if agentEditorTarget()} {#await import('$lib/components/flows/content/AgentEditorModal.svelte') then { default: AgentEditorModal }} - t.host === undefined} /> + t.host === undefined} + onRenamed={(from, to) => { + void loadResources() + // Only while the dialog still shows the agent: closed mid-request, it already cleared the + // anchor, and writing it back would reopen the editor on refresh. + if (agentEditorTarget()?.path !== from) return + // Claimed first, as a row click does, so the deep-link effect does not reopen it. + handledHash = `#/resource/${to}` + setPageDrawerAnchor(RESOURCES_PATH, to) + }} + /> {/await} {/if} From 73dc892f9c9c840a5f0fb12fcbb28bbe0f38795f Mon Sep 17 00:00:00 2001 From: hugocasa Date: Wed, 16 Sep 2026 10:45:34 +0200 Subject: [PATCH 39/44] fix: walk the whole fork ancestry for app installations and fork conflicts (#11151) * fix: walk the whole fork ancestry for app installations and fork conflicts Co-Authored-By: Claude Fable 5.1 * docs: describe the fork-conflict gate as ancestor-wide Co-Authored-By: Claude Fable 5.1 * chore: update ee-repo-ref to d252afcc80e77fcc4f9a2a346b80908c8605a6c0 This commit updates the EE repository reference after PR #803 was merged in windmill-ee-private. Previous ee-repo-ref: 5f68c8c351ffc92feccffe69a857b60be376464e New ee-repo-ref: d252afcc80e77fcc4f9a2a346b80908c8605a6c0 Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Fable 5.1 Co-authored-by: windmill-internal-app[bot] --- backend/ee-repo-ref.txt | 2 +- .../tests/fixtures/schedule_fork_conflict.sql | 46 ++++++++ backend/tests/git_sync_fork_credential.rs | 100 +++++++++++++++++- backend/tests/schedule_fork_conflict.rs | 59 +++++++++++ backend/windmill-api-schedule/src/lib.rs | 46 +++----- backend/windmill-common/src/workspaces.rs | 42 ++++++++ backend/windmill-trigger/src/handler.rs | 76 ++++--------- .../lib/components/ForkConflictModal.svelte | 16 +-- frontend/src/lib/stores.ts | 2 +- frontend/src/lib/utils/forkConflict.ts | 15 +-- 10 files changed, 301 insertions(+), 103 deletions(-) create mode 100644 backend/tests/fixtures/schedule_fork_conflict.sql create mode 100644 backend/tests/schedule_fork_conflict.rs diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 5d28f73b5a..317ca12c5e 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -93433d7c9dc34f2c0f56a5297d3453aabd2f9472 \ No newline at end of file +d252afcc80e77fcc4f9a2a346b80908c8605a6c0 diff --git a/backend/tests/fixtures/schedule_fork_conflict.sql b/backend/tests/fixtures/schedule_fork_conflict.sql new file mode 100644 index 0000000000..8e81d9ede6 --- /dev/null +++ b/backend/tests/fixtures/schedule_fork_conflict.sql @@ -0,0 +1,46 @@ +-- A three-deep fork chain whose schedule rows were cloned down at fork time. +-- The middle fork has since deleted its copy, so the leaf's schedule shares +-- its cron only with the root — the shape a direct-parent check misses. + +INSERT INTO workspace (id, name, owner, parent_workspace_id) VALUES + ('sfc-root', 'sfc-root', 'sfc-admin', NULL), + ('sfc-mid', 'sfc-mid', 'sfc-admin', 'sfc-root'), + ('sfc-leaf', 'sfc-leaf', 'sfc-admin', 'sfc-mid'); + +INSERT INTO workspace_key (workspace_id, kind, key) VALUES + ('sfc-root', 'cloud', 'sfc-root-key'), + ('sfc-mid', 'cloud', 'sfc-mid-key'), + ('sfc-leaf', 'cloud', 'sfc-leaf-key'); + +INSERT INTO workspace_settings (workspace_id) VALUES + ('sfc-root'), ('sfc-mid'), ('sfc-leaf'); + +INSERT INTO group_ (workspace_id, name, summary, extra_perms) VALUES + ('sfc-root', 'all', 'All users', '{}'), + ('sfc-mid', 'all', 'All users', '{}'), + ('sfc-leaf', 'all', 'All users', '{}'); + +INSERT INTO password(email, password_hash, login_type, super_admin, verified, name, username) + VALUES ('sfc-admin@windmill.dev', 'x', 'password', true, true, 'SFC Admin', 'sfc-admin'); + +INSERT INTO usr(workspace_id, email, username, is_admin, role) VALUES + ('sfc-root', 'sfc-admin@windmill.dev', 'sfc-admin', true, 'Admin'), + ('sfc-mid', 'sfc-admin@windmill.dev', 'sfc-admin', true, 'Admin'), + ('sfc-leaf', 'sfc-admin@windmill.dev', 'sfc-admin', true, 'Admin'); + +INSERT INTO token(token_hash, token_prefix, token, email, label, super_admin) + VALUES (encode(sha256('SFC_ADMIN_TOKEN'::bytea), 'hex'), 'SFC_ADMIN_', 'SFC_ADMIN_TOKEN', 'sfc-admin@windmill.dev', 't', true); + +-- Enabling pushes the next run, which needs the scheduled script to exist. +INSERT INTO script (workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES + ('sfc-leaf', 'sfc-admin', 'export async function main() { return "ok" }', '{}', '', '', 'f/shared/job', 7788001, 'deno', ''); + +INSERT INTO schedule (workspace_id, path, edited_by, edited_at, schedule, enabled, script_path, args, is_flow, email, timezone, extra_perms, permissioned_as) +VALUES + ('sfc-root', 'f/shared/nightly', 'sfc-admin', NOW(), '0 0 0 * * *', true, 'f/shared/job', '{}', false, 'sfc-admin@windmill.dev', 'UTC', '{}', 'u/sfc-admin'), + ('sfc-leaf', 'f/shared/nightly', 'sfc-admin', NOW(), '0 0 0 * * *', false, 'f/shared/job', '{}', false, 'sfc-admin@windmill.dev', 'UTC', '{}', 'u/sfc-admin'), + -- A path only the leaf has: nothing above shares it. + ('sfc-leaf', 'f/shared/own', 'sfc-admin', NOW(), '0 0 0 * * *', false, 'f/shared/job', '{}', false, 'sfc-admin@windmill.dev', 'UTC', '{}', 'u/sfc-admin'); + +GRANT ALL PRIVILEGES ON TABLE workspace_key TO windmill_admin; +GRANT ALL PRIVILEGES ON TABLE workspace_key TO windmill_user; diff --git a/backend/tests/git_sync_fork_credential.rs b/backend/tests/git_sync_fork_credential.rs index 10cf1e3735..9d5eb62dfb 100644 --- a/backend/tests/git_sync_fork_credential.rs +++ b/backend/tests/git_sync_fork_credential.rs @@ -12,8 +12,8 @@ use sqlx::{Pool, Postgres}; use windmill_common::git_sync_ee::{ - create_repo_webhook, git_credential_for_url, repo_provider, repo_supports_managed_git_features, - set_git_credential, GitProvider, + create_repo_webhook, git_app_installations_for, git_credential_for_url, managed_pr_base_branch, + repo_provider, repo_supports_managed_git_features, set_git_credential, GitProvider, }; use windmill_common::workspaces::GitCredentialProvider; @@ -278,3 +278,99 @@ async fn an_unreachable_gitlab_host_is_the_reported_error( ); Ok(()) } + +/// GitHub App installations are normally copied into a fork, but a workspace +/// attached as a dev workspace, or forked before its parent connected the App, +/// holds none, and neither does anything forked from it. The lookup reaches the +/// nearest workspace up the chain that holds some, and the background App path +/// (PR base resolution here) authenticates with that installation's token. +#[sqlx::test(fixtures("git_sync_fork_credential"))] +async fn app_installations_come_from_the_nearest_ancestor_holding_some( + db: Pool, +) -> anyhow::Result<()> { + use axum::{routing::get, Router}; + use std::sync::{Arc, Mutex}; + + // A stand-in GitHub API: one repository, and a record of who asked for it. + let seen: Arc>> = Arc::new(Mutex::new(vec![])); + let app = Router::new().route( + "/api/v3/repos/acme/repo", + get({ + let seen = seen.clone(); + move |headers: axum::http::HeaderMap| { + let seen = seen.clone(); + async move { + let auth = headers + .get("authorization") + .and_then(|v| v.to_str().ok()) + .unwrap_or("") + .to_string(); + seen.lock().unwrap().push(auth); + axum::Json(serde_json::json!({ "default_branch": "trunk" })) + } + } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let port = listener.local_addr()?.port(); + tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + let stub = format!("http://127.0.0.1:{port}"); + + // The root holds the installation, with a cached token so nothing is minted. + sqlx::query( + "UPDATE workspace_settings SET git_app_installations = $1::jsonb WHERE workspace_id = 'parent-ws'", + ) + .bind(serde_json::json!([{ + "installation_id": 42, "account_id": "acme", "jwt_token": "x", + "github_base_url": stub, + "installation_token": "root-token", "installation_token_expiration": 4102444800i64 + }])) + .execute(&db) + .await?; + // The fork's copy of the resource names the App-backed repository. + sqlx::query("UPDATE resource SET value = $1::jsonb WHERE workspace_id = 'deep-fork-ws' AND path = 'u/admin/repo'") + .bind(serde_json::json!({ "url": format!("{stub}/acme/repo.git"), "is_github_app": true })) + .execute(&db) + .await?; + + assert_eq!( + git_app_installations_for(&db, "deep-fork-ws").await?, + ("parent-ws".to_string(), vec![(42, Some(stub.clone()))]), + "two levels down, the root's installations are the ones to use" + ); + assert_eq!( + git_app_installations_for(&db, "orphan-ws").await?, + ("orphan-ws".to_string(), vec![]), + "a workspace with nothing above it resolves nothing" + ); + assert_eq!( + managed_pr_base_branch(&db, "deep-fork-ws", REPO) + .await? + .as_deref(), + Some("trunk"), + "the background App path reaches the repository through the root's installation" + ); + let seen = seen.lock().unwrap().clone(); + assert!( + !seen.is_empty() && seen.iter().all(|auth| auth == "Bearer root-token"), + "every call authenticated with the root's cached token: {seen:?}" + ); + + // A closer holder takes precedence over the root. + sqlx::query( + "UPDATE workspace_settings SET git_app_installations = $1::jsonb WHERE workspace_id = 'fork-ws'", + ) + .bind(serde_json::json!([{ + "installation_id": 7, "account_id": "acme", "jwt_token": "x", + "github_base_url": stub, + "installation_token": "mid-token", "installation_token_expiration": 4102444800i64 + }])) + .execute(&db) + .await?; + assert_eq!( + git_app_installations_for(&db, "deep-fork-ws").await?.0, + "fork-ws", + "the nearest holder wins over the root" + ); + Ok(()) +} diff --git a/backend/tests/schedule_fork_conflict.rs b/backend/tests/schedule_fork_conflict.rs new file mode 100644 index 0000000000..9a7ae8ff6d --- /dev/null +++ b/backend/tests/schedule_fork_conflict.rs @@ -0,0 +1,59 @@ +//! Enabling a schedule in a fork warns about every ancestor sharing the path, +//! not only the direct parent: the row was cloned down the whole chain, so the +//! cron is shared with whichever ancestors still hold a copy. + +use serde_json::json; +use sqlx::{Pool, Postgres}; + +use windmill_test_utils::*; + +async fn set_enabled( + base: &str, + path: &str, + enabled: bool, + force: bool, +) -> anyhow::Result<(u16, String)> { + let resp = reqwest::Client::new() + .post(format!("{base}/api/w/sfc-leaf/schedules/setenabled/{path}")) + .header("Authorization", "Bearer SFC_ADMIN_TOKEN") + .json(&json!({ "enabled": enabled, "force": force })) + .send() + .await?; + Ok((resp.status().as_u16(), resp.text().await?)) +} + +#[sqlx::test(fixtures("schedule_fork_conflict"))] +async fn enabling_in_a_fork_names_the_nearest_ancestor_sharing_the_path( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let base = format!("http://localhost:{}", server.addr.port()); + + let (status, body) = set_enabled(&base, "f/shared/nightly", true, false).await?; + assert_eq!(status, 400, "{body}"); + assert!( + body.contains("fork-conflict:schedule:sfc-root"), + "the middle fork deleted its copy, so the root is the one still sharing the cron: {body}" + ); + + let (status, body) = set_enabled(&base, "f/shared/own", true, false).await?; + assert_eq!( + status, 200, + "a path nothing upstream has enables freely: {body}" + ); + + sqlx::query( + "INSERT INTO schedule (workspace_id, path, edited_by, edited_at, schedule, enabled, script_path, args, is_flow, email, timezone, extra_perms, permissioned_as) + VALUES ('sfc-mid', 'f/shared/nightly', 'sfc-admin', NOW(), '0 0 0 * * *', false, 'f/shared/job', '{}', false, 'sfc-admin@windmill.dev', 'UTC', '{}', 'u/sfc-admin')", + ) + .execute(&db) + .await?; + let (status, body) = set_enabled(&base, "f/shared/nightly", true, false).await?; + assert_eq!(status, 400, "{body}"); + assert!( + body.contains("fork-conflict:schedule:sfc-mid"), + "with the parent holding a copy again, it is the nearest and gets named: {body}" + ); + Ok(()) +} diff --git a/backend/windmill-api-schedule/src/lib.rs b/backend/windmill-api-schedule/src/lib.rs index d581608c10..dfd4a6e3ba 100644 --- a/backend/windmill-api-schedule/src/lib.rs +++ b/backend/windmill-api-schedule/src/lib.rs @@ -1129,35 +1129,23 @@ pub async fn set_enabled( check_scopes(&authed, || format!("schedules:write:{}", path))?; reject_reserved_schedule_path(path)?; - // Block enabling a schedule in a fork when the parent has the same path - // (regardless of parent's enabled flag), unless force=true. Two enabled - // crons fire in lockstep; even when the parent is currently disabled the - // user is likely to re-enable it later, at which point both fire — better - // to surface that risk at every fork-side enable. There's no namespacing - // fix for schedules (Phase 3 doesn't help cron); the user has to confirm - // or point the script at fork-only side effects. + // Block enabling a schedule in a fork when an ancestor has the same path + // (regardless of its enabled flag), unless force=true. Two enabled crons + // fire in lockstep; even when the ancestor is currently disabled the user + // is likely to re-enable it later, at which point both fire — better to + // surface that risk at every fork-side enable. There's no namespacing fix + // for schedules (Phase 3 doesn't help cron); the user has to confirm or + // point the script at fork-only side effects. if payload.enabled && !payload.force { - let parent_id: Option = sqlx::query_scalar!( - "SELECT parent_workspace_id FROM workspace WHERE id = $1", - &w_id + if let Some(ancestor_id) = windmill_common::workspaces::nearest_fork_ancestor_having( + &mut *tx, "schedule", &w_id, path, ) - .fetch_optional(&mut *tx) .await? - .flatten(); - if let Some(parent_id) = parent_id { - let exists: Option = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM schedule WHERE workspace_id = $1 AND path = $2)", - &parent_id, - path, - ) - .fetch_one(&mut *tx) - .await?; - if exists == Some(true) { - return Err(Error::BadRequest(format!( - "fork-conflict:schedule:{}", - parent_id - ))); - } + { + return Err(Error::BadRequest(format!( + "fork-conflict:schedule:{}", + ancestor_id + ))); } } let before = trigger_history::snapshot_row(&mut *tx, "schedule", &w_id, path).await?; @@ -1699,9 +1687,9 @@ pub use windmill_queue::schedule::clear_schedule; #[derive(Deserialize)] pub struct SetEnabled { pub enabled: bool, - /// Bypass the parent-state warning when enabling a schedule in a fork - /// whose parent has the same path enabled. The frontend sets this after - /// the user confirms the duplicate-firing dialog. + /// Bypass the fork-conflict warning when enabling a schedule in a fork + /// while an ancestor workspace has the same path. The frontend sets this + /// after the user confirms the duplicate-firing dialog. #[serde(default)] pub force: bool, } diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index 9b6a471b2b..7cf910ef84 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -1788,6 +1788,48 @@ pub async fn workspace_with_fork_ancestors(db: &crate::DB, w_id: &str) -> Result Ok(chain) } +/// The nearest fork ancestor of `w_id` holding a row at `path` in `table`, or `None` when no +/// ancestor does (or `w_id` is not a fork). Fork creation clones trigger and schedule rows down +/// the whole chain, so a row shares its upstream identifier (Kafka group, PG slot, cron) with +/// every ancestor that still has one, not just the direct parent, which may have deleted its +/// copy since. +/// +/// Runs on the caller's connection so it sees the caller's transaction, and uncached: the +/// answer depends on the target table, not only on lineage. +/// +/// `table` is interpolated into SQL, hence `'static`: a trigger's `TABLE_NAME` or a literal, +/// never caller input. Reads lineage for any `w_id` with no authorization check, like +/// [`fork_ancestor_chain`], so the caller must already be authorized for `w_id`. +pub async fn nearest_fork_ancestor_having( + conn: &mut sqlx::PgConnection, + table: &'static str, + w_id: &str, + path: &str, +) -> Result> { + sqlx::query_scalar(&format!( + r#" + WITH RECURSIVE chain AS ( + SELECT id, parent_workspace_id, 0 AS depth + FROM workspace WHERE id = $1 + UNION ALL + SELECT w.id, w.parent_workspace_id, chain.depth + 1 + FROM workspace w + JOIN chain ON w.id = chain.parent_workspace_id + WHERE chain.depth < 20 + ) + SELECT chain.id FROM chain + JOIN {table} t ON t.workspace_id = chain.id AND t.path = $2 + WHERE chain.depth > 0 + ORDER BY chain.depth LIMIT 1 + "# + )) + .bind(w_id) + .bind(path) + .fetch_optional(&mut *conn) + .await + .map_err(|e| Error::internal_err(format!("resolving fork ancestors of {w_id}: {e:#}"))) +} + lazy_static::lazy_static! { /// workspace id -> (root workspace id, expiry ts). Read once per job start, so correctness /// rests on the invalidation rather than on the TTL: every mutation that can change the answer diff --git a/backend/windmill-trigger/src/handler.rs b/backend/windmill-trigger/src/handler.rs index 953f3a571b..66d6770354 100644 --- a/backend/windmill-trigger/src/handler.rs +++ b/backend/windmill-trigger/src/handler.rs @@ -97,14 +97,14 @@ pub trait TriggerCrud: Send + Sync + 'static { const DEPLOYMENT_NAME: &'static str; const ADDITIONAL_SELECT_FIELDS: &[&'static str] = &[]; const IS_ALLOWED_ON_CLOUD: bool; - /// Whether enabling this trigger in a fork while the parent has the same - /// path enabled is a real conflict (shared upstream resource). True for + /// Whether enabling this trigger in a fork while an ancestor workspace has + /// the same path is a real conflict (shared upstream resource). True for /// listener-based kinds where two consumers compete (Kafka group, PG slot, /// SQS queue, etc.) and for Websocket where both subscribers fire on every /// broadcast. False for kinds whose upstream identifier is implicitly /// workspace-scoped at runtime (HTTP routes, Email local_part — clones for /// the non-workspaced sub-case are filtered out, so any cloned row is - /// already collision-free vs. the parent). + /// already collision-free vs. its ancestors). const FORK_CONFLICT_ON_ENABLE: bool = true; fn get_deployed_object(path: String, parent_path: Option) -> DeployedObject; @@ -1095,54 +1095,14 @@ async fn exists_trigger( #[derive(serde::Deserialize)] struct SetTriggerModePayload { mode: TriggerMode, - /// When true, bypass the parent-state warning that would otherwise reject - /// enabling a trigger that's already enabled in the parent workspace. - /// The frontend sets this after the user confirms the duplicate-execution - /// dialog. See windmill-trigger/src/handler.rs::set_trigger_mode for the - /// full check. + /// When true, bypass the fork-conflict warning that would otherwise reject + /// enabling a trigger an ancestor workspace also has at this path. The + /// frontend sets this after the user confirms the duplicate-execution + /// dialog. See `set_trigger_mode` for the full check. #[serde(default)] force: bool, } -/// Returns the parent workspace id when this workspace is a fork *and* the -/// parent has a row at the same trigger path. Used to gate enabling a trigger -/// in a fork behind an explicit `force=true` confirmation: the fork's row was -/// cloned from the parent, so its upstream identifier (Kafka group, PG slot, -/// SQS queue URL, etc.) is shared by construction. The risk is independent of -/// the parent's current `mode`: if the parent is enabled, the two listeners -/// compete; if it's disabled, the fork can destructively take over shared -/// state (e.g. advance the PG WAL, claim an MQTT client_id) before the parent -/// re-enables. Either way, the user should be asked to confirm. -async fn parent_has_trigger( - tx: &mut PgConnection, - table_name: &str, - workspace_id: &str, - path: &str, -) -> Result> { - let parent: Option = - sqlx::query_scalar("SELECT parent_workspace_id FROM workspace WHERE id = $1") - .bind(workspace_id) - .fetch_optional(&mut *tx) - .await? - .flatten(); - let Some(parent_id) = parent else { - return Ok(None); - }; - let exists: Option = sqlx::query_scalar(&format!( - "SELECT EXISTS(SELECT 1 FROM {} WHERE workspace_id = $1 AND path = $2)", - table_name - )) - .bind(&parent_id) - .bind(path) - .fetch_one(&mut *tx) - .await?; - Ok(if exists == Some(true) { - Some(parent_id) - } else { - None - }) -} - async fn set_trigger_mode( Extension(handler): Extension>, authed: ApiAuthed, @@ -1157,22 +1117,28 @@ async fn set_trigger_mode( let mut tx = user_db.begin(&authed).await?; // Block transitioning a trigger in a fork to any mode that attaches a - // listener (Enabled or Suspended) when the parent has the same path, + // listener (Enabled or Suspended) when an ancestor has the same path, // unless the caller passes force=true. Suspended still keeps the // listener attached — it just stops auto-running queued jobs — so a // suspended fork would still split Kafka events / share a PG slot - // with the parent. The cloned upstream identifier is shared by - // construction; the risk is independent of the parent's current mode. - // Skipped for kinds where the upstream identifier is already - // workspace-scoped at runtime (HTTP, Email). + // with the ancestor. The risk is independent of the ancestor's current + // mode: enabled, the two listeners compete; disabled, the fork can + // destructively take over shared state (advance the PG WAL, claim an + // MQTT client_id) before it re-enables. Skipped for kinds where the + // upstream identifier is already workspace-scoped at runtime (HTTP, Email). if T::FORK_CONFLICT_ON_ENABLE && payload.mode != TriggerMode::Disabled && !payload.force { - if let Some(parent_id) = - parent_has_trigger(&mut *tx, T::TABLE_NAME, &workspace_id, path).await? + if let Some(ancestor_id) = windmill_common::workspaces::nearest_fork_ancestor_having( + &mut *tx, + T::TABLE_NAME, + &workspace_id, + path, + ) + .await? { return Err(Error::BadRequest(format!( "fork-conflict:{}:{}", T::TRIGGER_TYPE, - parent_id + ancestor_id ))); } } diff --git a/frontend/src/lib/components/ForkConflictModal.svelte b/frontend/src/lib/components/ForkConflictModal.svelte index e0c360320d..03afd71407 100644 --- a/frontend/src/lib/components/ForkConflictModal.svelte +++ b/frontend/src/lib/components/ForkConflictModal.svelte @@ -32,29 +32,29 @@ close(true)} onCanceled={() => close(false)} > {#if state}

    - The parent workspace ({state.parentWorkspaceId}) has the same - {state.kindLabel} configured at this path. Because this fork's row was cloned from it, the upstream - identifier is shared. + The upstream workspace ({state.upstreamWorkspaceId}) this fork + descends from has the same {state.kindLabel} configured at this path. Because this fork's row was + cloned from it, the upstream identifier is shared.

    {#if family === 'split'} If both are enabled, the two listeners will compete on the same upstream and each side will receive only a fraction of its events. {:else if family === 'duplicate'} - If both are enabled, every event will fire the script twice — once in the fork and once in - the parent. + If both are enabled, every event will fire the script twice: once in the fork and once in + the upstream workspace. {:else if family === 'slot'} The cloned replication_slot_name points at the same Postgres slot, which only allows one consumer at a time. Enabling here will either fail with "slot already active" - if the parent is enabled, or hijack the slot's WAL position if it isn't — causing the parent - to lose events when re-enabled. + if the upstream workspace is enabled, or hijack the slot's WAL position if it isn't, causing + it to lose events when re-enabled. {:else} Enabling it here may compete for the same upstream events or duplicate side effects. {/if} diff --git a/frontend/src/lib/stores.ts b/frontend/src/lib/stores.ts index 229ecff7ad..a3147b171c 100644 --- a/frontend/src/lib/stores.ts +++ b/frontend/src/lib/stores.ts @@ -242,7 +242,7 @@ export type GlobalForkModalState = { export type ForkConflictModalState = { kind: string kindLabel: string - parentWorkspaceId: string + upstreamWorkspaceId: string resolve: (proceed: boolean) => void } diff --git a/frontend/src/lib/utils/forkConflict.ts b/frontend/src/lib/utils/forkConflict.ts index d903790b44..5572e01f51 100644 --- a/frontend/src/lib/utils/forkConflict.ts +++ b/frontend/src/lib/utils/forkConflict.ts @@ -2,14 +2,15 @@ import { forkConflictModal } from '$lib/stores' /** * The backend rejects "enable" requests on triggers/schedules in a fork when - * the parent workspace has the same path enabled. The error body is shaped as - * `fork-conflict::` + * an upstream workspace (the parent, or an ancestor further up) has the same + * path. The error body is shaped as + * `fork-conflict::` * so the UI can show a tailored confirm-to-proceed dialog and re-issue the * call with `force: true` if the user agrees. */ export interface ForkConflict { kind: string - parentWorkspaceId: string + upstreamWorkspaceId: string } export function detectForkConflict(e: unknown): ForkConflict | null { @@ -20,7 +21,7 @@ export function detectForkConflict(e: unknown): ForkConflict | null { : ((body as any)?.error?.message ?? (body as any)?.message ?? (e as any)?.message ?? '') const m = String(raw).match(/fork-conflict:([^:]+):(.+)/) if (!m) return null - return { kind: m[1], parentWorkspaceId: m[2].trim() } + return { kind: m[1], upstreamWorkspaceId: m[2].trim() } } /** @@ -30,11 +31,11 @@ export function detectForkConflict(e: unknown): ForkConflict | null { * on two rows in quick succession), resolve the older promise to false so * the prior caller doesn't hang. */ -function askForkConflictConfirm(kind: string, kindLabel: string, parentWorkspaceId: string) { +function askForkConflictConfirm(kind: string, kindLabel: string, upstreamWorkspaceId: string) { return new Promise((resolve) => { const previous = forkConflictModal.val previous?.resolve(false) - forkConflictModal.val = { kind, kindLabel, parentWorkspaceId, resolve } + forkConflictModal.val = { kind, kindLabel, upstreamWorkspaceId, resolve } }) } @@ -64,7 +65,7 @@ export async function withForkConflictRetry( const proceed = await askForkConflictConfirm( conflict.kind, kindLabel, - conflict.parentWorkspaceId + conflict.upstreamWorkspaceId ) // User explicitly dismissed the modal — treat as a silent no-op so the // caller's catch block doesn't pop a redundant error toast. From b51c0eabbe774c78a9cbf824a3df6528d970b8f6 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Wed, 16 Sep 2026 10:46:15 +0200 Subject: [PATCH 40/44] feat: stream reasoning summaries in AI agent Responses API steps (#11124) * feat: stream reasoning summaries in AI agent Responses API steps Co-Authored-By: Claude Opus 5 * fix: retry without a reasoning summary a strict gateway rejects Co-Authored-By: Claude Opus 5 * fix: request the reasoning summary whether or not the step streams Co-Authored-By: Claude Opus 5 * feat: return the OpenAI reasoning summary in the agent step's reasoning Co-Authored-By: Claude Opus 5 * fix: retry without a reasoning summary a gateway rejects with 422 Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Opus 5 --- .../windmill-ai/src/providers/anthropic.rs | 1 + backend/windmill-ai/src/providers/openai.rs | 194 ++++++++++++++++-- backend/windmill-ai/src/query_builder.rs | 3 + backend/windmill-ai/src/sse.rs | 63 ++++++ backend/windmill-ai/src/types.rs | 3 +- backend/windmill-worker/src/ai_executor.rs | 23 ++- 6 files changed, 269 insertions(+), 18 deletions(-) diff --git a/backend/windmill-ai/src/providers/anthropic.rs b/backend/windmill-ai/src/providers/anthropic.rs index 038aef2c2e..0bb7316dcb 100644 --- a/backend/windmill-ai/src/providers/anthropic.rs +++ b/backend/windmill-ai/src/providers/anthropic.rs @@ -902,6 +902,7 @@ mod tests { attachments: None, has_websearch: false, prompt_cache_key: None, + reasoning_summary: false, }; AnthropicQueryBuilder::new(AIProvider::Anthropic, platform) diff --git a/backend/windmill-ai/src/providers/openai.rs b/backend/windmill-ai/src/providers/openai.rs index 8fa65dca21..14c9fdb4d3 100644 --- a/backend/windmill-ai/src/providers/openai.rs +++ b/backend/windmill-ai/src/providers/openai.rs @@ -1,6 +1,7 @@ use crate::{ ai_providers::AIProvider, ai_types::OpenAIToolCall, + credentials::ProviderCredentials, image_handler::{prepare_messages_for_api, s3_object_to_content_part}, proxy::{build_openai_compatible_proxy_request, ProxyBuildArgs, ProxyRequest}, query_builder::{BuildRequestArgs, ParsedResponse, QueryBuilder, StreamEventSink}, @@ -11,7 +12,14 @@ use crate::{ use async_trait::async_trait; use serde::{Deserialize, Serialize}; use serde_json::value::RawValue; -use windmill_common::{client::AuthedClient, error::Error}; +use std::{ + collections::{BTreeMap, HashMap}, + hash::{DefaultHasher, Hash, Hasher}, + time::{Duration, Instant}, +}; +use windmill_common::{cache::Cache, client::AuthedClient, error::Error}; + +use super::REASONING_OFF_SENTINEL; // Responses API structures #[derive(Deserialize)] @@ -192,13 +200,79 @@ pub struct ResponsesApiTextFormat { pub format: ResponsesApiTextFormatConfig, } -/// Reasoning config for the Responses API (`reasoning: { effort }`). -/// The summary is intentionally not requested, mirroring the copilot chat: OpenAI -/// gates reasoning summaries behind organization verification, so asking for one -/// would fail the request for unverified orgs. +/// Reasoning config for the Responses API (`reasoning: { effort, summary }`). #[derive(Serialize)] pub struct ResponsesApiReasoning { pub effort: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub summary: Option, +} + +lazy_static::lazy_static! { + /// Refused reasoning summaries, so later requests skip asking instead of paying a + /// rejected call each. A refusal belongs to the organization the request bills or to the + /// model, so the key holds the model and everything that authenticates (the API key and + /// the resource headers, which can carry it instead). Entries expire as an org gets verified. + static ref REASONING_SUMMARY_UNAVAILABLE: Cache<(String, u64), Instant> = Cache::new(500); +} + +const REASONING_SUMMARY_UNAVAILABLE_TTL: Duration = Duration::from_secs(3600); + +fn reasoning_summary_cache_key( + base_url: &str, + model: &str, + api_key: Option<&str>, + custom_headers: &HashMap, +) -> (String, u64) { + let mut hasher = DefaultHasher::new(); + model.hash(&mut hasher); + api_key.hash(&mut hasher); + // Sorted: two maps with the same entries can iterate them in different orders. + custom_headers + .iter() + .collect::>() + .hash(&mut hasher); + (base_url.to_string(), hasher.finish()) +} + +fn credentials_cache_key(credentials: &ProviderCredentials, model: &str) -> (String, u64) { + reasoning_summary_cache_key( + &credentials.base_url, + model, + credentials.api_key.as_deref(), + &credentials.custom_headers, + ) +} + +/// Whether this model is known to be refused reasoning summaries with these credentials. +pub fn is_reasoning_summary_unavailable(credentials: &ProviderCredentials, model: &str) -> bool { + REASONING_SUMMARY_UNAVAILABLE + .get(&credentials_cache_key(credentials, model)) + .is_some_and(|learned_at| learned_at.elapsed() < REASONING_SUMMARY_UNAVAILABLE_TTL) +} + +/// Record that this model was refused a reasoning summary with these credentials. +pub fn remember_reasoning_summary_unavailable(credentials: &ProviderCredentials, model: &str) { + REASONING_SUMMARY_UNAVAILABLE.insert(credentials_cache_key(credentials, model), Instant::now()); +} + +/// Whether a rejected request was refused over its reasoning summary, e.g. `Your +/// organization must be verified to generate reasoning summaries` (param +/// `reasoning.summary`). An OpenAI-kind resource can also point at a gateway that validates +/// the body strictly and names only the unknown `summary` property. +pub fn rejects_reasoning_summary(status: u16, body: &str) -> bool { + // 422 is how FastAPI-based gateways reject a body that fails validation. + if !matches!(status, 400 | 403 | 422) { + return false; + } + let body = body.to_lowercase(); + let unknown_field = body.contains("additional properties are not allowed") + || body.contains("unrecognized request argument") + || body.contains("extra inputs are not permitted"); + body.contains("reasoning.summary") + || body.contains("verified to generate reasoning summar") + || body.contains("verified to stream reasoning summar") + || (unknown_field && body.contains("summary")) } #[derive(Serialize)] @@ -435,9 +509,13 @@ impl OpenAIQueryBuilder { tools, stream: Some(true), temperature: args.temperature, - reasoning: args - .reasoning_effort - .map(|effort| ResponsesApiReasoning { effort: effort.to_string() }), + reasoning: args.reasoning_effort.map(|effort| ResponsesApiReasoning { + effort: effort.to_string(), + // A request that does not reason has nothing to summarize, yet asking still + // gets an unverified organization's request rejected. + summary: (args.reasoning_summary && effort != REASONING_OFF_SENTINEL) + .then(|| "auto".to_string()), + }), max_output_tokens: args.max_tokens, text, prompt_cache_key: args.prompt_cache_key, @@ -538,9 +616,8 @@ impl QueryBuilder for OpenAIQueryBuilder { } else { Some(parser.accumulated_content) }, - // The Responses stream has no reasoning-summary event in - // `OpenAIResponsesSSEEvent`, so nothing thinks out loud on this path yet. - reasoning: None, + reasoning: (!parser.accumulated_reasoning.is_empty()) + .then_some(parser.accumulated_reasoning), tool_calls: parser.accumulated_tool_calls.into_values().collect(), events_str: Some(parser.events_str), annotations: parser.annotations, @@ -640,8 +717,11 @@ mod tests { } } - async fn build_text_body(messages: &[OpenAIMessage], system_prompt: Option<&str>) -> String { - let args = BuildRequestArgs { + fn text_args<'a>( + messages: &'a [OpenAIMessage], + system_prompt: Option<&'a str>, + ) -> BuildRequestArgs<'a> { + BuildRequestArgs { messages, tools: None, model: "gpt-5", @@ -655,14 +735,98 @@ mod tests { attachments: None, has_websearch: false, prompt_cache_key: Some(PROMPT_CACHE_KEY), - }; + reasoning_summary: true, + } + } + async fn build_body(args: &BuildRequestArgs<'_>) -> String { OpenAIQueryBuilder::new(AIProvider::OpenAI) - .build_request(&args, &client(), "test-workspace") + .build_request(args, &client(), "test-workspace") .await .unwrap() } + async fn build_text_body(messages: &[OpenAIMessage], system_prompt: Option<&str>) -> String { + build_body(&text_args(messages, system_prompt)).await + } + + async fn reasoning_of(effort: Option<&str>, reasoning_summary: bool) -> serde_json::Value { + let messages = vec![message("user", "hi")]; + let args = BuildRequestArgs { + reasoning_effort: effort, + reasoning_summary, + ..text_args(&messages, None) + }; + let request: serde_json::Value = serde_json::from_str(&build_body(&args).await).unwrap(); + request["reasoning"].clone() + } + + #[tokio::test] + async fn requests_a_reasoning_summary_only_when_the_model_reasons() { + assert_eq!( + reasoning_of(Some("high"), true).await, + serde_json::json!({ "effort": "high", "summary": "auto" }) + ); + assert_eq!( + reasoning_of(Some("none"), true).await, + serde_json::json!({ "effort": "none" }) + ); + assert_eq!( + reasoning_of(Some("high"), false).await, + serde_json::json!({ "effort": "high" }) + ); + assert!(reasoning_of(None, true).await.is_null()); + } + + #[test] + fn recognizes_a_refused_reasoning_summary() { + let unverified = r#"{"error":{"message":"Your organization must be verified to generate reasoning summaries. Please go to: https://platform.openai.com/settings/organization/general and click on Verify Organization.","type":"invalid_request_error","param":"reasoning.summary","code":"unsupported_value"}}"#; + assert!(rejects_reasoning_summary(400, unverified)); + assert!(!rejects_reasoning_summary(500, unverified)); + assert!(rejects_reasoning_summary( + 400, + r#"{"detail":"Additional properties are not allowed ('summary' was unexpected)"}"# + )); + assert!(rejects_reasoning_summary( + 422, + r#"{"detail":[{"type":"extra_forbidden","loc":["body","reasoning","summary"],"msg":"Extra inputs are not permitted","input":"auto"}]}"# + )); + assert!(!rejects_reasoning_summary( + 400, + r#"{"error":{"message":"Invalid 'prompt_cache_key': string too long","param":"prompt_cache_key"}}"# + )); + } + + /// A resource can authenticate through `headers` with no API key: one organization's + /// refusal must not withhold summaries from another's. + #[test] + fn keys_a_refused_summary_by_the_header_credential() { + let headers = |pairs: &[(&str, &str)]| { + pairs + .iter() + .map(|(name, value)| (name.to_string(), value.to_string())) + .collect::>() + }; + let url = "https://api.openai.com/v1"; + let org_a = headers(&[("Authorization", "Bearer org-a"), ("X-Trace", "1")]); + let org_a_reordered = headers(&[("X-Trace", "1"), ("Authorization", "Bearer org-a")]); + let org_b = headers(&[("Authorization", "Bearer org-b"), ("X-Trace", "1")]); + + assert_ne!( + reasoning_summary_cache_key(url, "gpt-5", None, &org_a), + reasoning_summary_cache_key(url, "gpt-5", None, &org_b) + ); + assert_eq!( + reasoning_summary_cache_key(url, "gpt-5", None, &org_a), + reasoning_summary_cache_key(url, "gpt-5", None, &org_a_reordered) + ); + // A model can refuse summaries that another model on the same credentials streams. + assert_ne!( + reasoning_summary_cache_key(url, "gpt-5", None, &org_a), + reasoning_summary_cache_key(url, "gpt-5-mini", None, &org_a) + ); + } + /// The worker prepends the system prompt as a system message *and* passes it as /// `system_prompt`; the request must still carry it exactly once. #[tokio::test] diff --git a/backend/windmill-ai/src/query_builder.rs b/backend/windmill-ai/src/query_builder.rs index 60ece300a3..3281b51cd0 100644 --- a/backend/windmill-ai/src/query_builder.rs +++ b/backend/windmill-ai/src/query_builder.rs @@ -27,6 +27,9 @@ pub struct BuildRequestArgs<'a> { /// the prefix (the step), never from the request. `None` retries a key the /// endpoint rejected. pub prompt_cache_key: Option<&'a str>, + /// Ask for a summary of the model's reasoning where the provider streams one. + /// `false` once the provider refused summaries to these credentials. + pub reasoning_summary: bool, } /// Response from AI provider diff --git a/backend/windmill-ai/src/sse.rs b/backend/windmill-ai/src/sse.rs index 986642fd7e..054baeb5de 100644 --- a/backend/windmill-ai/src/sse.rs +++ b/backend/windmill-ai/src/sse.rs @@ -815,6 +815,14 @@ pub enum OpenAIResponsesSSEEvent { #[serde(rename = "response.output_text.annotation.added")] AnnotationAdded { annotation: OpenAIUrlCitationEvent }, + /// A new reasoning summary part starts (only sent when `reasoning.summary` was requested) + #[serde(rename = "response.reasoning_summary_part.added")] + ReasoningSummaryPartAdded {}, + + /// Reasoning summary text delta + #[serde(rename = "response.reasoning_summary_text.delta")] + ReasoningSummaryTextDelta { delta: String }, + /// Catch-all for unknown event types #[serde(other)] Other, @@ -823,6 +831,8 @@ pub enum OpenAIResponsesSSEEvent { /// OpenAI Responses API SSE Parser for streaming responses pub struct OpenAIResponsesSSEParser { pub accumulated_content: String, + /// The reasoning summary streamed before the answer, kept so it can be stored with it. + pub accumulated_reasoning: String, pub accumulated_tool_calls: HashMap, /// Maps item_id -> (name, call_id) for function calls tool_call_metadata: HashMap, @@ -836,12 +846,15 @@ pub struct OpenAIResponsesSSEParser { pub used_websearch: bool, /// Token usage from response.completed event pub usage: Option, + /// Reasoning summary parts seen so far, to separate them as paragraphs + reasoning_summary_parts: usize, } impl OpenAIResponsesSSEParser { pub fn new(stream_event_processor: Box) -> Self { Self { accumulated_content: String::new(), + accumulated_reasoning: String::new(), accumulated_tool_calls: HashMap::new(), tool_call_metadata: HashMap::new(), tool_call_arguments: HashMap::new(), @@ -850,6 +863,7 @@ impl OpenAIResponsesSSEParser { annotations: Vec::new(), used_websearch: false, usage: None, + reasoning_summary_parts: 0, } } } @@ -959,6 +973,28 @@ impl SSEParser for OpenAIResponsesSSEParser { } } + OpenAIResponsesSSEEvent::ReasoningSummaryPartAdded {} => { + self.reasoning_summary_parts += 1; + if self.reasoning_summary_parts > 1 { + self.accumulated_reasoning.push_str("\n\n"); + let event = + StreamingEvent::ReasoningTokenDelta { content: "\n\n".to_string() }; + self.stream_event_processor + .send(event, &mut self.events_str) + .await?; + } + } + + OpenAIResponsesSSEEvent::ReasoningSummaryTextDelta { delta } => { + if !delta.is_empty() { + self.accumulated_reasoning.push_str(&delta); + let event = StreamingEvent::ReasoningTokenDelta { content: delta }; + self.stream_event_processor + .send(event, &mut self.events_str) + .await?; + } + } + // Ignore other event types OpenAIResponsesSSEEvent::Done {} | OpenAIResponsesSSEEvent::Created {} @@ -1015,6 +1051,33 @@ mod tests { assert_eq!(token_usage.total_tokens, Some(4821)); } + struct ReasoningSink; + + #[async_trait::async_trait] + impl StreamEventSink for ReasoningSink { + async fn send(&self, event: StreamingEvent, events_str: &mut String) -> Result<(), Error> { + if let StreamingEvent::ReasoningTokenDelta { content } = event { + events_str.push_str(&content); + } + Ok(()) + } + } + + #[tokio::test] + async fn streams_openai_responses_reasoning_summary_parts_as_paragraphs() { + let mut parser = OpenAIResponsesSSEParser::new(Box::new(ReasoningSink)); + for data in [ + r#"{"type":"response.reasoning_summary_part.added","item_id":"rs_1","output_index":0,"summary_index":0,"part":{"type":"summary_text","text":""}}"#, + r#"{"type":"response.reasoning_summary_text.delta","item_id":"rs_1","output_index":0,"summary_index":0,"delta":"**Planning**"}"#, + r#"{"type":"response.reasoning_summary_part.added","item_id":"rs_1","output_index":0,"summary_index":1,"part":{"type":"summary_text","text":""}}"#, + r#"{"type":"response.reasoning_summary_text.delta","item_id":"rs_1","output_index":0,"summary_index":1,"delta":"Then answer"}"#, + ] { + parser.parse_event_data(data).await.unwrap(); + } + assert_eq!(parser.events_str, "**Planning**\n\nThen answer"); + assert_eq!(parser.accumulated_reasoning, "**Planning**\n\nThen answer"); + } + #[test] fn openai_delta_parses_reasoning_content() { // DeepSeek and similar stream reasoning under `reasoning_content`. diff --git a/backend/windmill-ai/src/types.rs b/backend/windmill-ai/src/types.rs index 8dfb9af1c6..ae85718e8a 100644 --- a/backend/windmill-ai/src/types.rs +++ b/backend/windmill-ai/src/types.rs @@ -377,8 +377,7 @@ pub struct AIAgentResult<'a> { /// The model's thinking across every iteration of the loop, in order, blank-line /// separated. Present whenever the provider's parser surfaced any, whether or not /// the step streams, so a downstream step never has to pick it out of `wm_stream`. - /// Absent when the model thought nothing, and on the OpenAI Responses path, whose - /// parser does not return reasoning yet. + /// Absent when the model thought nothing. #[serde(skip_serializing_if = "Option::is_none")] pub reasoning: Option, #[serde(skip_serializing_if = "Option::is_none")] diff --git a/backend/windmill-worker/src/ai_executor.rs b/backend/windmill-worker/src/ai_executor.rs index debfbeff39..156369e095 100644 --- a/backend/windmill-worker/src/ai_executor.rs +++ b/backend/windmill-worker/src/ai_executor.rs @@ -26,6 +26,10 @@ use windmill_ai::{ image_handler::upload_image_to_s3, providers::{ create_chat_completions_query_builder, create_query_builder, is_chat_completions_only, + openai::{ + is_reasoning_summary_unavailable, rejects_reasoning_summary, + remember_reasoning_summary_unavailable, + }, remember_chat_completions_only, }, proxy::{ @@ -1300,6 +1304,10 @@ pub async fn run_agent( attachments: args.user_attachments.as_deref(), has_websearch, prompt_cache_key: include_prompt_cache_key.then_some(prompt_cache_key.as_str()), + reasoning_summary: !is_reasoning_summary_unavailable( + &credentials, + args.provider.get_model(), + ), }; // A worker cannot run the client credentials exchange, so an OAuth resource @@ -1350,7 +1358,8 @@ pub async fn run_agent( // An endpoint can reject the request shape rather than the model: // `stream_options` and `prompt_cache_key`, which not every OpenAI-compatible - // gateway accepts, and the route itself, when an Azure resource is outside + // gateway accepts, a reasoning summary, which OpenAI refuses to unverified + // organizations, and the route itself, when an Azure resource is outside // the Responses API's model/region matrix. Each is retried once with that // part dropped. // Set where the route is found to be absent, and read once the fallback has @@ -1407,6 +1416,9 @@ pub async fn run_agent( && status.as_u16() == 400 && text.contains("prompt_cache_key"); + let summary_refused = build_args.reasoning_summary + && rejects_reasoning_summary(status.as_u16(), &text); + // Only the first call of the step may re-route: an endpoint that // does not serve this API rejects that one already, whereas a // rejection once the conversation is under way is about the @@ -1430,6 +1442,15 @@ pub async fn run_agent( ); include_prompt_cache_key = false; build_args.prompt_cache_key = None; + } else if summary_refused { + tracing::info!( + "Retrying request without the reasoning summary the endpoint refused" + ); + remember_reasoning_summary_unavailable( + &credentials, + args.provider.get_model(), + ); + build_args.reasoning_summary = false; } else if route_unserved { tracing::info!( "Endpoint rejected the request ({}), falling back to chat/completions", From 54553b2add6941395f03ca34ee24cf335a3b23c2 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Wed, 16 Sep 2026 10:53:24 +0200 Subject: [PATCH 41/44] fix(cli): resolve lockgen imports through modules a push leaves alone (#11160) Co-authored-by: Claude Opus 5 --- cli/src/commands/sync/sync.ts | 42 ++++++++++ cli/src/utils/dependency_tree.ts | 81 ++++++++++++++++++- cli/test/dependency_tree_unit.test.ts | 64 ++++++++++++++- .../sync_push_auto_metadata_repro.test.ts | 75 +++++++++++++++++ 4 files changed, 259 insertions(+), 3 deletions(-) diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index 38b9e87f84..138a346594 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -128,6 +128,8 @@ import { } from "../../utils/metadata.ts"; import { DoubleLinkedDependencyTree, + LocalScripts, + resolvePlaceholdersFromLocal, uploadScripts, } from "../../utils/dependency_tree.ts"; import { @@ -176,6 +178,7 @@ import { isDbtModulePath, isDbtGeneratedPath, isModuleEntryPoint, + scriptPathToRemotePath, getScriptBasePathFromModulePath, hasWrongFormatSuffix, DBT_DESCRIPTOR_NAME, @@ -3385,6 +3388,37 @@ async function addToChangedIfNotExists(p: string, tracker: ChangeTracker) { } } +/** + * Index the checkout's standalone scripts by the remote path a relative import + * resolves to, reusing the content the local/remote diff already read. + * + * Same classification as `addToChangedIfNotExists`: a flow or app inline script + * is not addressable as an import target, and a module bundle is addressed by + * its entry point. + */ +function localScriptsByRemotePath( + localMap: Record, +): LocalScripts { + const byRemotePath: LocalScripts = new Map(); + for (const [p, content] of Object.entries(localMap)) { + if (isScriptModulePath(p)) { + if (!isModuleEntryPoint(p)) continue; + } else if ( + !hasScriptExt(p) || + isDatatableMigrationPath(p) || + isFileResource(p) || + isFilesetResource(p) || + isFlowPath(p) || + isAppPath(p) || + isRawAppPath(p) + ) { + continue; + } + byRemotePath.set(scriptPathToRemotePath(p), { localPath: p, content }); + } + return byRemotePath; +} + export async function buildTracker(changes: Change[]) { const tracker: ChangeTracker = { scripts: [], @@ -5074,6 +5108,14 @@ export async function push( } if (autoRegenerate && tree) { + // Pass 1 only ever walks the change set, so anything imported through a + // module the push leaves alone is still a dead end here. + await resolvePlaceholdersFromLocal( + tree, + localScriptsByRemotePath(localMap), + opts.defaultTs, + ); + // Propagate staleness through imports + upload script content to // raw_script_temp so the dep job can resolve cross-folder relative imports // via temp_script_refs (instead of hitting 404s for not-yet-deployed diff --git a/cli/src/utils/dependency_tree.ts b/cli/src/utils/dependency_tree.ts index b0b75e659e..959a77b478 100644 --- a/cli/src/utils/dependency_tree.ts +++ b/cli/src/utils/dependency_tree.ts @@ -5,7 +5,7 @@ import { Workspace } from "../commands/workspace/workspace.ts"; import * as wmill from "../../gen/services.gen.ts"; import type { ScriptLang } from "../../gen/types.gen.ts"; -import { ScriptLanguage } from "./script_common.ts"; +import { ScriptLanguage, inferContentTypeFromFilePath } from "./script_common.ts"; import { filterWorkspaceDependencies, generateScriptHash, @@ -14,6 +14,65 @@ import { updateMetadataGlobalLock, } from "./metadata.ts"; import { generateHash } from "./utils.ts"; +import { extractRelativeImports } from "./relative_imports.ts"; + +/** A local script file, keyed in `LocalScripts` by its Windmill remote path. */ +export interface LocalScriptSource { + localPath: string; + content: string; +} +export type LocalScripts = Map; + +/** + * Give every import target that is only a placeholder its local content and its + * own imports, so the graph continues through it. A tree seeded from a subset of + * the checkout otherwise dead-ends at any module outside that subset — a + * re-export barrel needing no edit, typically — hiding what it re-exports from + * `getTempScriptRefs`, which then resolves it against the deployed copy. + */ +export async function resolvePlaceholdersFromLocal( + tree: DoubleLinkedDependencyTree, + localScripts: LocalScripts, + defaultTs: "bun" | "deno" | undefined +): Promise { + // A module resolved here can expose placeholders of its own (a barrel behind + // a barrel), so keep going until a round resolves nothing. + for (;;) { + let resolved = false; + for (const remotePath of tree.placeholderPaths()) { + const local = localScripts.get(remotePath); + if (!local) continue; + let language: ScriptLanguage; + try { + language = inferContentTypeFromFilePath(local.localPath, defaultTs); + } catch { + // A bare `.sql` names no dialect, so its imports cannot be read here. + continue; + } + const imports = await extractRelativeImports( + local.content, + remotePath, + language + ); + // Never directly stale: it is outside the change set, so nothing relocks + // it. It is here to carry edges, and to be uploaded if it differs from + // what is deployed. + await tree.addNode( + remotePath, + local.content, + language, + "", + imports, + "script", + remotePath, + local.localPath, + false + ); + resolved = true; + } + if (!resolved) break; + } +} /** * Diff local scripts against deployed versions, upload only those that differ. @@ -97,6 +156,9 @@ interface DependencyNode { originalPath: string; // Original path passed to handler (with extension for scripts) isRawApp?: boolean; // Only set for apps isDirectlyStale: boolean; // True if this item's content changed (vs transitively stale) + // True while the node exists only because something imports it, so it carries + // no content and no imports of its own. + isPlaceholder: boolean; } export class DoubleLinkedDependencyTree { @@ -130,9 +192,11 @@ export class DoubleLinkedDependencyTree { content: "", stalenessHash: "", language: "deno", metadata: "", imports: new Set(), importedBy: new Set(), itemType: "script", folder: "", originalPath: "", isDirectlyStale: false, + isPlaceholder: true, }); } const node = this.nodes.get(path)!; + node.isPlaceholder = false; node.content = content; node.stalenessHash = stalenessHash; node.language = language; @@ -155,7 +219,7 @@ export class DoubleLinkedDependencyTree { stalenessHash: "", language: depsInfo?.language ?? "deno", metadata: "", imports: new Set(), importedBy: new Set(), itemType: "dependencies", folder: "", originalPath: depsPath, - isDirectlyStale: !isUpToDate, + isDirectlyStale: !isUpToDate, isPlaceholder: false, }); } } @@ -169,6 +233,7 @@ export class DoubleLinkedDependencyTree { content: "", stalenessHash: "", language: "deno", metadata: "", imports: new Set(), importedBy: new Set(), itemType: "script", folder: "", originalPath: "", isDirectlyStale: false, + isPlaceholder: true, }); } this.nodes.get(importPath)!.importedBy.add(path); @@ -309,6 +374,18 @@ export class DoubleLinkedDependencyTree { return this.nodes.keys(); } + /** + * Paths that exist only as somebody's import target, so the traversal stops + * at them instead of continuing into what they themselves import. + */ + placeholderPaths(): string[] { + const result: string[] = []; + for (const [path, node] of this.nodes.entries()) { + if (node.isPlaceholder) result.push(path); + } + return result; + } + /** * Returns paths of all stale nodes (those with a staleReason). */ diff --git a/cli/test/dependency_tree_unit.test.ts b/cli/test/dependency_tree_unit.test.ts index 1a92bb9894..8e9c733c00 100644 --- a/cli/test/dependency_tree_unit.test.ts +++ b/cli/test/dependency_tree_unit.test.ts @@ -2,7 +2,10 @@ import { expect, test } from "bun:test"; import { mkdtemp, rm } from "node:fs/promises"; import os from "node:os"; import * as path from "node:path"; -import { DoubleLinkedDependencyTree } from "../src/utils/dependency_tree.ts"; +import { + DoubleLinkedDependencyTree, + resolvePlaceholdersFromLocal, +} from "../src/utils/dependency_tree.ts"; // addNode consults wmill-lock.yaml from cwd for workspace deps; run inside a // temp dir so the test never reads/writes the repo's own lock file. @@ -104,3 +107,62 @@ test("getAllTempScriptRefs is a superset of getTempScriptRefs for any node", asy }); }); }); + +// Two barrels deep so the fixpoint matters: resolving the first one is what +// puts the second in the tree, and only a further round reaches the leaf. +test("resolvePlaceholdersFromLocal walks the graph through unresolved barrels", async () => { + await withTempDir(async () => { + const tree = new DoubleLinkedDependencyTree(); + await tree.addNode( + "f/app/consumer", + `import { subtract } from "../barrel/index.ts"`, + "bun", + "", + ["f/barrel/index"], + "script", + "f/app/consumer", + "f/app/consumer.ts", + true, + ); + await tree.addNode( + "f/barrel/helper", + "export function subtract(a: number, b: number) { return a - b }", + "bun", + "", + [], + "script", + "f/barrel/helper", + "f/barrel/helper.ts", + true, + ); + // Neither barrel is in the change set, so both are bare import targets. + expect(tree.getTempScriptRefs("f/app/consumer")).toEqual({}); + + await resolvePlaceholdersFromLocal( + tree, + new Map([ + [ + "f/barrel/index", + { + localPath: "f/barrel/index.ts", + content: `export * from "./mid.ts"`, + }, + ], + [ + "f/barrel/mid", + { + localPath: "f/barrel/mid.ts", + content: `export * from "./helper.ts"`, + }, + ], + ]), + "bun", + ); + + // Only the leaf diverged from deployed, so only it was uploaded. + tree.setContentHash("f/barrel/helper", "hash_helper"); + expect(tree.getTempScriptRefs("f/app/consumer")).toEqual({ + "f/barrel/helper": "hash_helper", + }); + }); +}); diff --git a/cli/test/sync_push_auto_metadata_repro.test.ts b/cli/test/sync_push_auto_metadata_repro.test.ts index 4854198d64..38da094a37 100644 --- a/cli/test/sync_push_auto_metadata_repro.test.ts +++ b/cli/test/sync_push_auto_metadata_repro.test.ts @@ -268,3 +268,78 @@ test( }); }, ); + +// The importer and the leaf change; the barrel between them does not, so it is +// absent from the push's change set. See `resolvePlaceholdersFromLocal`. +test( + "sync push --auto-metadata succeeds when a changed leaf sits behind an unchanged barrel", + { timeout: 180000 }, + async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile(`${tempDir}/wmill.yaml`, wmillYaml); + + await createLocalScript( + tempDir, + "f/barrel", + "helper", + "bun", + `export function add(a: number, b: number) { return a + b; }\n`, + ); + await createLocalScript( + tempDir, + "f/barrel", + "index", + "bun", + `export * from "./helper.ts";\n`, + ); + await createLocalScript( + tempDir, + "f/app", + "consumer", + "bun", + `import { add } from "../barrel/index.ts"; +export async function main() { return add(1, 2); } +`, + ); + + const deploy = await backend.runCLICommand( + ["sync", "push", "--yes", "--auto-metadata"], + tempDir, + ); + if (deploy.code !== 0) { + console.log("STDOUT:", deploy.stdout); + console.log("STDERR:", deploy.stderr); + } + expect(deploy.code).toBe(0); + + // Add an export to the leaf and use it from the importer. The barrel + // re-exports it already, so it stays byte-identical and out of the push. + await writeFile( + `${tempDir}/f/barrel/helper.ts`, + `export function add(a: number, b: number) { return a + b; } +export function subtract(a: number, b: number) { return a - b; } +`, + ); + await writeFile( + `${tempDir}/f/app/consumer.ts`, + `import { subtract } from "../barrel/index.ts"; +export async function main() { return subtract(3, 1); } +`, + ); + + const result = await backend.runCLICommand( + ["sync", "push", "--yes", "--auto-metadata"], + tempDir, + ); + if (result.code !== 0) { + console.log("STDOUT:", result.stdout); + console.log("STDERR:", result.stderr); + } + expect(result.code).toBe(0); + + const combined = result.stdout + result.stderr; + expect(combined).not.toContain("No matching export"); + expect(combined).not.toContain("Failed to generate lockfile"); + }); + }, +); From 49d0310ecc08040b6c6fa4f402584543d679e2e8 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 16 Sep 2026 12:03:32 +0200 Subject: [PATCH 42/44] fix(apps): re-check access in place after a password sign-in (#11166) * fix(apps): re-check access in place after a password sign-in Co-Authored-By: Claude Opus 5 (1M context) * style: drop redundant comment in the password sign-in hand-back Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Opus 5 (1M context) --- frontend/src/lib/components/Login.svelte | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/frontend/src/lib/components/Login.svelte b/frontend/src/lib/components/Login.svelte index da032717f6..44ba7e4e15 100644 --- a/frontend/src/lib/components/Login.svelte +++ b/frontend/src/lib/components/Login.svelte @@ -61,6 +61,10 @@ popup?: boolean firstTime?: boolean autoRedirect?: boolean + /** Supplying this replaces the post-login redirect: the card hands back instead of + * navigating, and the host is expected to re-check access in place. For a gate on a + * page that stays mounted, `rd` is the URL already shown, so navigating there would + * re-run nothing. */ onLoginSuccess?: () => void /** A refusal the popup relayed back, in the server's words. */ onLoginError?: (message: string) => void @@ -290,6 +294,11 @@ // Finally, we check whether the user is a superadmin refreshSuperadmin() + + if (onLoginSuccess) { + onLoginSuccess() + return + } redirectUser() } From 6885226c2686ee2d44d87b75107493aca41cf5dd Mon Sep 17 00:00:00 2001 From: Guilhem Date: Wed, 16 Sep 2026 12:05:22 +0200 Subject: [PATCH 43/44] collapse detail page header actions into menus below lg (#11163) Co-authored-by: Claude Fable 5.1 --- .../src/lib/components/DropdownV2Inner.svelte | 17 ++- .../components/common/button/Button.svelte | 2 + .../details/DetailPageHeader.svelte | 119 +++++++++++++----- .../details/ErrorHandlerToggleButton.svelte | 53 ++------ .../components/details/errorHandlerToggle.ts | 39 ++++++ frontend/src/lib/utils.ts | 2 + frontend/src/lib/utils/editInFork.ts | 13 ++ .../(logged)/flows/get/[...path]/+page.svelte | 7 ++ .../scripts/get/[...hash]/+page.svelte | 7 ++ 9 files changed, 179 insertions(+), 80 deletions(-) create mode 100644 frontend/src/lib/components/details/errorHandlerToggle.ts diff --git a/frontend/src/lib/components/DropdownV2Inner.svelte b/frontend/src/lib/components/DropdownV2Inner.svelte index 5f8b96c32f..7749310b5b 100644 --- a/frontend/src/lib/components/DropdownV2Inner.svelte +++ b/frontend/src/lib/components/DropdownV2Inner.svelte @@ -53,12 +53,17 @@ {#if item.icon} {/if} -

    - {item.displayName} -

    +
    +

    + {item.displayName} +

    + {#if item.description} +

    {item.description}

    + {/if} +
    {@render item.extra?.()} {#if item.shortcut || item.selected || item.toggle !== undefined}