From 943ef6eb2089f4b744cfa7945ce47f7f3b361ec7 Mon Sep 17 00:00:00 2001 From: centdix <40307056+centdix@users.noreply.github.com> Date: Mon, 1 Jun 2026 19:48:50 +0200 Subject: [PATCH 01/61] feat: add workspace datatable tools to global AI chat mode (#9395) * feat: add workspace datatable tools to global AI chat mode Co-Authored-By: Claude Opus 4.8 (1M context) * test: cover global-mode datatable tools pure logic Co-Authored-By: Claude Opus 4.8 (1M context) * feat: expose datatable SQL SDK reference via get_instructions in global mode Co-Authored-By: Claude Opus 4.8 (1M context) * feat: make datatable get_instructions language-aware, default TypeScript Co-Authored-By: Claude Opus 4.8 (1M context) * refactor: drop datatable/whitelist args from global init_app tool Co-Authored-By: Claude Opus 4.8 (1M context) * feat: flag missing datatable config as an explicit blocking error in global mode Co-Authored-By: Claude Opus 4.8 (1M context) * refactor: drop dead branch in exec_datatable_sql result handling Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Claude Opus 4.8 (1M context) --- .../copilot/chat/datatableTools.test.ts | 200 +++++++++++++ .../components/copilot/chat/datatableTools.ts | 278 ++++++++++++++++++ .../copilot/chat/global/core.test.ts | 22 ++ .../components/copilot/chat/global/core.ts | 81 +++-- system_prompts/auto-generated/index.d.ts | 2 +- system_prompts/auto-generated/index.ts | 18 +- system_prompts/generate.py | 20 +- 7 files changed, 583 insertions(+), 38 deletions(-) create mode 100644 frontend/src/lib/components/copilot/chat/datatableTools.test.ts create mode 100644 frontend/src/lib/components/copilot/chat/datatableTools.ts diff --git a/frontend/src/lib/components/copilot/chat/datatableTools.test.ts b/frontend/src/lib/components/copilot/chat/datatableTools.test.ts new file mode 100644 index 0000000000..ff933eae20 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/datatableTools.test.ts @@ -0,0 +1,200 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { listMock, schemaMock, runSqlMock } = vi.hoisted(() => ({ + listMock: vi.fn(), + schemaMock: vi.fn(), + runSqlMock: vi.fn() +})) + +vi.mock('./shared', () => ({ + createToolDef: (_schema: unknown, name: string, description: string) => ({ + type: 'function', + function: { name, description, parameters: {} } + }) +})) + +vi.mock('$lib/gen', () => ({ + WorkspaceService: { + listDataTableTables: listMock, + getDataTableTableSchema: schemaMock + } +})) + +vi.mock('$lib/components/jobs/utils', () => ({ + runScriptAndPollResult: runSqlMock +})) + +import { getDatatableTools } from './datatableTools' + +function createToolCallbacks() { + return { + setToolStatus: vi.fn(), + removeToolStatus: vi.fn() + } +} + +function getTool(name: string) { + const tool = getDatatableTools().find((entry) => entry.def.function.name === name) + if (!tool) { + throw new Error(`${name} tool not found`) + } + return tool +} + +function run(name: string, args: Record = {}) { + return getTool(name).fn({ + args, + workspace: 'test-workspace', + helpers: {}, + toolCallbacks: createToolCallbacks(), + toolId: `tool-${name}` + }) +} + +beforeEach(() => { + listMock.mockReset() + schemaMock.mockReset() + runSqlMock.mockReset() +}) + +describe('list_datatables', () => { + it('aggregates the table count across schemas and returns metadata verbatim', async () => { + const metadata = [ + { datatable_name: 'main', schemas: { public: ['users', 'orders'], analytics: ['events'] } }, + { datatable_name: 'warehouse', schemas: { public: ['facts'] } } + ] + listMock.mockResolvedValue(metadata) + + const tool = getTool('list_datatables') + const callbacks = createToolCallbacks() + const result = await tool.fn({ + args: {}, + workspace: 'test-workspace', + helpers: {}, + toolCallbacks: callbacks, + toolId: 'tool-list' + }) + + expect(listMock).toHaveBeenCalledWith({ workspace: 'test-workspace' }) + expect(JSON.parse(result)).toEqual(metadata) + // 2 + 1 + 1 = 4 tables across 2 datatables + expect(callbacks.setToolStatus).toHaveBeenCalledWith('tool-list', { + content: 'Listed 2 datatable(s) with 4 table(s)' + }) + }) + + it('explains that configuring a datatable is a blocking prerequisite when none exist', async () => { + listMock.mockResolvedValue([]) + const result = await run('list_datatables') + expect(result).toContain('No datatables are configured in this workspace') + expect(result).toContain('Workspace settings → Data Tables') + expect(result).toContain('blocked') + expect(result).toContain('Do not call exec_datatable_sql') + }) + + it('surfaces backend errors as a readable message', async () => { + listMock.mockRejectedValue(new Error('boom')) + const result = await run('list_datatables') + expect(result).toContain('Error listing datatables: boom') + }) +}) + +describe('get_datatable_table_schema', () => { + it('returns the columns for one table', async () => { + schemaMock.mockResolvedValue({ + datatable_name: 'main', + schema_name: 'public', + table_name: 'users', + columns: { id: 'int4', email: 'text' } + }) + + const result = await run('get_datatable_table_schema', { + datatable_name: 'main', + schema_name: 'public', + table_name: 'users' + }) + + expect(schemaMock).toHaveBeenCalledWith({ + workspace: 'test-workspace', + datatableName: 'main', + schemaName: 'public', + tableName: 'users' + }) + expect(JSON.parse(result)).toEqual({ + datatable_name: 'main', + schema_name: 'public', + table_name: 'users', + columns: { id: 'int4', email: 'text' } + }) + }) +}) + +describe('exec_datatable_sql', () => { + it('requires confirmation', () => { + expect(getTool('exec_datatable_sql').requiresConfirmation).toBe(true) + }) + + it('returns all rows when the result is at or below the cap', async () => { + const rows = Array.from({ length: 100 }, (_, i) => ({ id: i })) + runSqlMock.mockResolvedValue(rows) + + const result = await run('exec_datatable_sql', { + datatable_name: 'main', + sql: 'SELECT * FROM t' + }) + + const parsed = JSON.parse(result) + expect(parsed.success).toBe(true) + expect(parsed.rowCount).toBe(100) + expect(parsed.result).toHaveLength(100) + expect(parsed.note).toBeUndefined() + expect(runSqlMock).toHaveBeenCalledWith({ + workspace: 'test-workspace', + requestBody: { + language: 'postgresql', + content: 'SELECT * FROM t', + args: { database: 'datatable://main' } + } + }) + }) + + it('truncates results above the cap and reports the full count', async () => { + const rows = Array.from({ length: 150 }, (_, i) => ({ id: i })) + runSqlMock.mockResolvedValue(rows) + + const parsed = JSON.parse(await run('exec_datatable_sql', { datatable_name: 'main', sql: 'SELECT 1' })) + expect(parsed.rowCount).toBe(150) + expect(parsed.result).toHaveLength(100) + expect(parsed.note).toBe('Showing first 100 of 150 rows') + }) + + it('treats a non-array result (e.g. DDL) as zero rows', async () => { + runSqlMock.mockResolvedValue(undefined) + const parsed = JSON.parse( + await run('exec_datatable_sql', { + datatable_name: 'main', + sql: 'CREATE TABLE t (id serial primary key)' + }) + ) + expect(parsed).toEqual({ success: true, rowCount: 0, result: [] }) + }) + + it('returns a failure object when the SQL job errors', async () => { + runSqlMock.mockRejectedValue(new Error('syntax error')) + const parsed = JSON.parse(await run('exec_datatable_sql', { datatable_name: 'main', sql: 'SLECT' })) + expect(parsed).toEqual({ success: false, error: 'syntax error' }) + }) + + it('turns the backend "datatable not found" error into an actionable, blocking message', async () => { + runSqlMock.mockRejectedValue(new Error('Internal: datatable main not found @workspaces.rs:565:20')) + const parsed = JSON.parse( + await run('exec_datatable_sql', { datatable_name: 'main', sql: 'CREATE TABLE t (id int)' }) + ) + expect(parsed.success).toBe(false) + expect(parsed.error).toContain('not configured in this workspace') + expect(parsed.error).toContain('Workspace settings → Data Tables') + expect(parsed.error).toContain('do not retry') + // The raw internal error is not surfaced to the model. + expect(parsed.error).not.toContain('workspaces.rs') + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/datatableTools.ts b/frontend/src/lib/components/copilot/chat/datatableTools.ts new file mode 100644 index 0000000000..cebde9e97e --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/datatableTools.ts @@ -0,0 +1,278 @@ +import { z } from 'zod' +import { WorkspaceService } from '$lib/gen' +import type { DataTableTables } from '$lib/gen/types.gen' +import { runScriptAndPollResult } from '$lib/components/jobs/utils' +import { createToolDef, type Tool } from './shared' + +/** + * Workspace-scoped datatable tools, with no app whitelist and no creation policy. + * + * Datatables are workspace-level managed PostgreSQL databases. The backend + * endpoints used here (`list_datatable_tables`, `get_datatable_table_schema`) + * and SQL execution (`datatable://`) are gated only by workspace + * membership, so these tools need no app context and operate directly on the + * workspace. This is the unrestricted counterpart to the app-mode datatable + * tools in `app/core.ts`, which additionally filter by the app's whitelist. + */ + +// ============= Utility ============= + +/** Memoize a factory function - the factory is only called once, on first access */ +const memo = (factory: () => T): (() => T) => { + let cached: T | undefined + return () => (cached ??= factory()) +} + +// ============= Pure workspace-scoped operations ============= + +/** List all datatables configured in the workspace, with their schema/table names. */ +export async function listDatatables(workspace: string): Promise { + return await WorkspaceService.listDataTableTables({ workspace }) +} + +/** Get the columns (column_name -> compact_type) of one datatable table. */ +export async function getDatatableColumns( + workspace: string, + datatableName: string, + schemaName: string, + tableName: string +): Promise> { + const schema = await WorkspaceService.getDataTableTableSchema({ + workspace, + datatableName, + schemaName, + tableName + }) + return schema.columns +} + +/** + * Execute an arbitrary SQL statement against a datatable. + * Supports SELECT/INSERT/UPDATE/DELETE as well as DDL (CREATE/ALTER/DROP). + * Returns rows for SELECT-like queries, an empty array otherwise. + */ +export async function execDatatableSql( + workspace: string, + datatableName: string, + sql: string +): Promise< + { success: true; result: Record[] } | { success: false; error: string } +> { + try { + const result = await runScriptAndPollResult({ + workspace, + requestBody: { + language: 'postgresql', + content: sql, + args: { database: `datatable://${datatableName}` } + } + }) + if (Array.isArray(result)) { + return { success: true, result } + } + return { success: true, result: [] } + } catch (e) { + return { success: false, error: e instanceof Error ? e.message : String(e) } + } +} + +// ============= Error helpers ============= + +/** + * The backend returns "datatable not found" when no datatable with that + * name is configured in the workspace settings. That is a hard, blocking + * prerequisite (not a transient failure), so we surface an explicit, actionable + * message instead of the raw internal error. + */ +function isDatatableNotConfiguredError(error: string | null | undefined): boolean { + return typeof error === 'string' && /datatable\s+\S+\s+not found/i.test(error) +} + +function datatableNotConfiguredMessage(datatableName: string): string { + return ( + `Datatable "${datatableName}" is not configured in this workspace, so this operation cannot run. ` + + `Datatables are not created by SQL — they must be set up first by the user in the workspace settings ` + + `(Workspace settings → Data Tables) before any table can be queried or created. ` + + `This is a required, blocking prerequisite: do not retry on this or another name. ` + + `Tell the user they need to configure a datatable (e.g. named "${datatableName}") in their workspace settings, then try again.` + ) +} + +const NO_DATATABLES_CONFIGURED_MESSAGE = + 'No datatables are configured in this workspace. Datatable operations (querying data, creating or altering tables) are blocked until a datatable exists. ' + + 'Datatables are not created by SQL — the user must set one up in the workspace settings (Workspace settings → Data Tables) first. ' + + 'Do not call exec_datatable_sql or assume a "main" datatable exists; instead tell the user this is a required prerequisite and ask them to configure a datatable in their workspace settings.' + +// ============= Tool definitions ============= + +const getListDatatablesSchema = memo(() => z.object({})) +const getListDatatablesToolDef = memo(() => + createToolDef( + getListDatatablesSchema(), + 'list_datatables', + 'List datatables configured in the workspace with schema and table names only. Does not include column definitions. Use this directly for table-list or available-tables summaries. Only call get_datatable_table_schema when column names/types are required.' + ) +) + +const getGetDatatableTableSchemaSchema = memo(() => + z.object({ + datatable_name: z.string().describe('The datatable name to inspect, e.g. "main".'), + schema_name: z.string().describe('The schema name, e.g. "public".'), + table_name: z.string().describe('The table name to inspect.') + }) +) +const getGetDatatableTableSchemaToolDef = memo(() => + createToolDef( + getGetDatatableTableSchemaSchema(), + 'get_datatable_table_schema', + 'Get column definitions for one datatable table. Do not call this for row counts or table-list summaries; list_datatables is enough for those.' + ) +) + +const getExecDatatableSqlSchema = memo(() => + z.object({ + datatable_name: z + .string() + .describe( + 'The name of the datatable to query (e.g., "main"). Must be one of the datatables configured in the workspace.' + ), + sql: z + .string() + .describe( + 'The SQL query to execute. Supports SELECT, INSERT, UPDATE, DELETE, CREATE TABLE, ALTER TABLE, DROP TABLE, etc. For SELECT queries, results are returned as an array of objects. A newly created table will appear in list_datatables automatically.' + ) + }) +) +const getExecDatatableSqlToolDef = memo(() => + createToolDef( + getExecDatatableSqlSchema(), + 'exec_datatable_sql', + 'Execute a SQL query on a workspace datatable. Use this to explore data, test queries, create/alter tables, or make changes. Creating a table is a normal CREATE TABLE statement — no registration step is needed.' + ) +) + +/** Maximum rows returned to the model for a SELECT query. */ +const MAX_ROWS = 100 + +/** + * The unrestricted workspace datatable tools, for registration in global mode. + * Helper-free: each tool reads `workspace` directly from the tool call params. + */ +export function getDatatableTools(): Tool<{}>[] { + return [ + { + def: getListDatatablesToolDef(), + fn: async ({ workspace, toolId, toolCallbacks }) => { + toolCallbacks.setToolStatus(toolId, { content: 'Listing datatables...' }) + try { + const metadata = await listDatatables(workspace) + if (metadata.length === 0) { + toolCallbacks.setToolStatus(toolId, { + content: 'No datatables configured — set one up in workspace settings' + }) + return NO_DATATABLES_CONFIGURED_MESSAGE + } + const totalTables = metadata.reduce( + (acc, datatable) => + acc + + Object.values(datatable.schemas).reduce((sum, tables) => sum + tables.length, 0), + 0 + ) + toolCallbacks.setToolStatus(toolId, { + content: `Listed ${metadata.length} datatable(s) with ${totalTables} table(s)` + }) + return JSON.stringify(metadata, null, 2) + } catch (e) { + const errorMsg = `Error listing datatables: ${e instanceof Error ? e.message : String(e)}` + toolCallbacks.setToolStatus(toolId, { content: errorMsg, error: errorMsg }) + return errorMsg + } + } + }, + { + def: getGetDatatableTableSchemaToolDef(), + fn: async ({ args, workspace, toolId, toolCallbacks }) => { + const parsedArgs = getGetDatatableTableSchemaSchema().parse(args) + toolCallbacks.setToolStatus(toolId, { + content: `Getting schema for ${parsedArgs.datatable_name}.${parsedArgs.schema_name}.${parsedArgs.table_name}...` + }) + try { + const columns = await getDatatableColumns( + workspace, + parsedArgs.datatable_name, + parsedArgs.schema_name, + parsedArgs.table_name + ) + toolCallbacks.setToolStatus(toolId, { + content: `Retrieved schema for ${parsedArgs.schema_name}.${parsedArgs.table_name}` + }) + return JSON.stringify( + { + datatable_name: parsedArgs.datatable_name, + schema_name: parsedArgs.schema_name, + table_name: parsedArgs.table_name, + columns + }, + null, + 2 + ) + } catch (e) { + const raw = e instanceof Error ? e.message : String(e) + const errorMsg = isDatatableNotConfiguredError(raw) + ? datatableNotConfiguredMessage(parsedArgs.datatable_name) + : `Error getting table schema: ${raw}` + toolCallbacks.setToolStatus(toolId, { content: errorMsg, error: errorMsg }) + return errorMsg + } + } + }, + { + def: getExecDatatableSqlToolDef(), + requiresConfirmation: true, + confirmationMessage: 'Execute SQL on datatable', + showDetails: true, + fn: async ({ args, workspace, toolId, toolCallbacks }) => { + const parsedArgs = getExecDatatableSqlSchema().parse(args) + toolCallbacks.setToolStatus(toolId, { + content: `Executing SQL on "${parsedArgs.datatable_name}"...` + }) + try { + const result = await execDatatableSql(workspace, parsedArgs.datatable_name, parsedArgs.sql) + if (result.success) { + // Successful runs always carry a `result` array (empty for DDL/DML), so + // SELECT rows and zero-row statements share one reporting path. + const rowCount = result.result.length + toolCallbacks.setToolStatus(toolId, { content: `Query returned ${rowCount} row(s)` }) + if (rowCount > MAX_ROWS) { + return JSON.stringify( + { + success: true, + rowCount, + result: result.result.slice(0, MAX_ROWS), + note: `Showing first ${MAX_ROWS} of ${rowCount} rows` + }, + null, + 2 + ) + } + return JSON.stringify({ success: true, rowCount, result: result.result }, null, 2) + } else { + const raw = result.error || 'Unknown error' + const errorMsg = isDatatableNotConfiguredError(raw) + ? datatableNotConfiguredMessage(parsedArgs.datatable_name) + : raw + toolCallbacks.setToolStatus(toolId, { content: `Error: ${errorMsg}`, error: errorMsg }) + return JSON.stringify({ success: false, error: errorMsg }) + } + } catch (e) { + const raw = e instanceof Error ? e.message : String(e) + const errorMsg = isDatatableNotConfiguredError(raw) + ? datatableNotConfiguredMessage(parsedArgs.datatable_name) + : raw + toolCallbacks.setToolStatus(toolId, { content: `Error: ${errorMsg}`, error: errorMsg }) + return JSON.stringify({ success: false, error: errorMsg }) + } + } + } + ] +} diff --git a/frontend/src/lib/components/copilot/chat/global/core.test.ts b/frontend/src/lib/components/copilot/chat/global/core.test.ts index 9521c970fb..c10efce9f6 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.test.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.test.ts @@ -209,6 +209,28 @@ describe('global AI tools', () => { vi.clearAllMocks() }) + it('defaults the datatable instruction subject to the TypeScript SQL SDK', async () => { + const result = await callGlobalTool('get_instructions', { subject: 'datatable' }) + expect(result).toContain('wmill.datatable(') + expect(result).toContain('TypeScript Datatable API') + expect(result).toContain('fetchOne') + // Defaults to TypeScript only — no Python noise. + expect(result).not.toContain('Python Datatable API') + }) + + it('returns only the requested language SDK for the datatable subject', async () => { + const ts = await callGlobalTool('get_instructions', { subject: 'datatable', language: 'bun' }) + expect(ts).toContain('TypeScript Datatable API') + expect(ts).not.toContain('Python Datatable API') + + const py = await callGlobalTool('get_instructions', { + subject: 'datatable', + language: 'python3' + }) + expect(py).toContain('Python Datatable API') + expect(py).not.toContain('TypeScript Datatable API') + }) + it('exposes hub search and path-aware test tools', () => { const names = globalTools.map((tool) => tool.def.function.name) diff --git a/frontend/src/lib/components/copilot/chat/global/core.ts b/frontend/src/lib/components/copilot/chat/global/core.ts index b30e30c9b0..db8ecb871f 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.ts @@ -48,7 +48,13 @@ import { validateEditableFlowJson } from '../flow/editableFlowJson' import { createInlineScriptSession } from '../flow/inlineScriptsUtils' -import { getFlowPrompt, getRawAppPrompt, getResourcePrompt, getScriptPrompt } from '$system_prompts' +import { + getDatatableSdkReference, + getFlowPrompt, + getRawAppPrompt, + getResourcePrompt, + getScriptPrompt +} from '$system_prompts' import type { ChatCompletionSystemMessageParam, ChatCompletionUserMessageParam @@ -66,6 +72,7 @@ import { type ToolDisplayAction } from '../shared' import type { ContextElement } from '../context' +import { getDatatableTools } from '../datatableTools' import { UserDraft, type UserDraftMeta } from '$lib/userDraft.svelte' import { emptySchema } from '$lib/utils' import { inferArgs } from '$lib/infer' @@ -118,6 +125,13 @@ const INSTRUCTION_SUBJECTS = [ 'resource', 'app' ] as const satisfies readonly WorkspaceItemType[] +// `datatable` is not a workspace item type, but the model can request the +// datatable SDK reference (the wmill.datatable() runnable API) the same way. +const INSTRUCTION_SUBJECTS_EXTRA = ['datatable'] as const +const ALL_INSTRUCTION_SUBJECTS = [ + ...INSTRUCTION_SUBJECTS, + ...INSTRUCTION_SUBJECTS_EXTRA +] as const const MAX_LIST_LIMIT = 100 type ActiveGlobalEditorType = Extract type LiveEditorDraftKind = Parameters[0] @@ -143,18 +157,18 @@ export type GlobalUserMessageOptions = { } const itemTypeSchema = z.enum(ITEM_TYPES) -const instructionSubjectSchema = z.enum(INSTRUCTION_SUBJECTS) +const instructionSubjectSchema = z.enum(ALL_INSTRUCTION_SUBJECTS) const triggerKindSchema = z.enum(TRIGGER_KINDS) const scriptLangSchema = z.enum($ScriptLang.enum) const getInstructionsSchema = z.object({ subject: instructionSubjectSchema.describe( - "The workspace item type to get authoring instructions for (script, flow, resource, app). Schedules, triggers, and variables don't need instructions — their tool schemas describe everything." + "What to get authoring instructions for: a workspace item type (script, flow, resource, app) or \"datatable\" for the wmill.datatable() SQL SDK used inside runnables. Schedules, triggers, and variables don't need instructions — their tool schemas describe everything." ), language: scriptLangSchema .optional() .describe( - 'Required when subject is script. Use the existing script language when modifying, or the requested target language when creating.' + 'The target language. Required when subject is script. For subject "datatable" it selects which SDK to return (e.g. "bun" for TypeScript, "python3" for Python) and defaults to TypeScript if omitted. Use the existing language when modifying, or the requested target language when creating. Other subjects ignore it.' ) }) @@ -558,20 +572,7 @@ const initAppSchema = z.object({ .enum(FRAMEWORK_KEYS) .describe( 'Frontend framework template. Confirm with the user before calling — never default silently. react19 is recommended for new apps.' - ), - data: z - .object({ - datatable: z.string().optional().describe('Default datatable name (e.g. "main").'), - schema: z.string().optional().describe('Default schema (PostgreSQL schema, optional).'), - tables: z - .array(z.string()) - .optional() - .describe( - 'Initially-whitelisted tables, in the format "/" or "/:
".' - ) - }) - .optional() - .describe('Optional datatable configuration. Omit unless the user asked to wire one up.') + ) }) const buildGlobalSystemPrompt = ( @@ -619,7 +620,14 @@ Raw apps: - Use write_app_file, patch_app_file, and delete_app_file for frontend files. - Use write_app_runnable and delete_app_runnable for backend runnables. - Use init_app only after confirming framework, path, and summary with the user. -- Use deploy_workspace_item after explicit user deploy intent; raw app deploy bundles JS/CSS before saving.` +- Use deploy_workspace_item after explicit user deploy intent; raw app deploy bundles JS/CSS before saving. + +Data Tables: +- Datatables are workspace-scoped managed PostgreSQL databases, shared across the workspace (not owned by any single app). They must be configured by the user in their workspace settings (Workspace settings → Data Tables); they cannot be created via SQL. +- Use list_datatables to discover the available datatables and their tables. Reuse an existing table rather than creating a duplicate. If list_datatables reports none, this is a blocking prerequisite — tell the user to set up a datatable in their workspace settings and stop; do not assume a "main" datatable exists or call exec_datatable_sql. +- Use get_datatable_table_schema only when you need a table's column names/types; list_datatables is enough for table-list or availability summaries. +- Use exec_datatable_sql to explore data, run queries, mutate rows, or change schema (CREATE/ALTER/DROP). Creating a table is a normal CREATE TABLE statement — it appears in list_datatables afterward, with no registration step. +- When writing runnable code (inline app runnables, scripts, flow modules) that reads or writes datatable data at runtime, it accesses a datatable via wmill.datatable(). Default to TypeScript (bun) unless the user asked for another language. Call get_instructions with subject "datatable" and language "bun" for the TypeScript SQL SDK reference (or language "python3" for Python) — it returns only that language so you get just what you need.` const DEFAULT_LIST_TYPES = ['script', 'flow'] as const satisfies readonly WorkspaceItemType[] @@ -1311,7 +1319,7 @@ function getFlowInstructions(): string { ${getFlowPrompt()}` } -type InstructionSubject = (typeof INSTRUCTION_SUBJECTS)[number] +type InstructionSubject = (typeof ALL_INSTRUCTION_SUBJECTS)[number] function getAppInstructions(): string { return `# Global draft app instructions @@ -1350,6 +1358,19 @@ function getResourceInstructions(): string { ${getResourcePrompt()}` } +function getDatatableInstructions(language?: ScriptLang): string { + // Default to the TypeScript SDK so we return only what's needed, not both. + const lang = language ?? 'bun' + return `# Datatable SQL SDK reference + +Datatables are workspace-scoped managed PostgreSQL databases. In chat, explore and shape them with the \`list_datatables\`, \`get_datatable_table_schema\`, and \`exec_datatable_sql\` tools. The reference below is for code you author inside runnables (inline app runnables, scripts, or flow rawscript modules) that reads or writes datatable data at runtime. + +- A runnable accesses a datatable via \`wmill.datatable()\` (the default "main") or \`wmill.datatable('')\`, referencing tables as \`schema.table\`. +- Use parameterized queries (the tagged template in TypeScript, \`$1\`/\`$2\` placeholders in Python) — never interpolate untrusted values into SQL strings. + +${getDatatableSdkReference(lang)}` +} + function getInstructions(subject: InstructionSubject, language?: ScriptLang): string { switch (subject) { case 'script': @@ -1360,6 +1381,8 @@ function getInstructions(subject: InstructionSubject, language?: ScriptLang): st return getResourceInstructions() case 'app': return getAppInstructions() + case 'datatable': + return getDatatableInstructions(language) } } @@ -1368,7 +1391,7 @@ export const globalTools: Tool<{}>[] = [ def: createToolDef( getInstructionsSchema, 'get_instructions', - 'Get authoring guidance for scripts, flows, resources, or apps.' + 'Get authoring guidance for scripts, flows, resources, apps, or the datatable SQL SDK (wmill.datatable()) used inside runnables.' ), fn: async ({ args, toolId, toolCallbacks }) => { const parsed = getInstructionsSchema.parse(args) @@ -1876,7 +1899,9 @@ export const globalTools: Tool<{}>[] = [ 'Check whether the side-panel preview is open in this AI session and which item (kind + path) it is showing. Call this before offering or calling open_preview so you do not re-open a preview that is already showing the item you just edited. Only meaningful inside a session.' ), fn: async (ctx) => getSessionPreviewStatus(sessionIdFromCtx(ctx)) - } + }, + // Workspace-scoped datatable tools (unrestricted: no whitelist, no creation policy) + ...getDatatableTools() ] // Tools that only make sense inside an AI session (they drive the session's @@ -2708,12 +2733,11 @@ async function initApp( path: string summary?: string framework: FrameworkKey - data?: { datatable?: string; schema?: string; tables?: string[] } }, ctx: WriteDraftCtx ): Promise { const { workspace, toolId, toolCallbacks } = ctx - const { path, summary, framework, data } = args + const { path, summary, framework } = args if (getGlobalDraft(workspace, 'app', path)) { throw new Error( @@ -2734,14 +2758,7 @@ async function initApp( const value: AppDraftValue = { summary, files: { ...template }, - runnables: { [STARTER_RUNNABLE_KEY]: { ...STARTER_RUNNABLE } }, - ...(data && { - data: { - tables: data.tables ?? [], - datatable: data.datatable, - schema: data.schema - } - }) + runnables: { [STARTER_RUNNABLE_KEY]: { ...STARTER_RUNNABLE } } } await recomputeAppPolicy(value) const stored = saveAppDraft(workspace, path, value) diff --git a/system_prompts/auto-generated/index.d.ts b/system_prompts/auto-generated/index.d.ts index 6018d7ef92..f7d8d632b4 100644 --- a/system_prompts/auto-generated/index.d.ts +++ b/system_prompts/auto-generated/index.d.ts @@ -3,5 +3,5 @@ export declare function getScriptPrompt(language: string): string; export declare function getFlowPrompt(): string; export declare function getResourcePrompt(): string; export declare function getRawAppPrompt(): string; -export declare function getDatatableSdkReference(): string; +export declare function getDatatableSdkReference(language?: string): string; export declare function getWorkflowAsCodePrompt(language?: string): string; diff --git a/system_prompts/auto-generated/index.ts b/system_prompts/auto-generated/index.ts index bbd90e1066..6ee3e20f4a 100644 --- a/system_prompts/auto-generated/index.ts +++ b/system_prompts/auto-generated/index.ts @@ -54,8 +54,22 @@ export function getRawAppPrompt(): string { return prompts.RAW_APP_BASE; } -// Helper to get datatable SDK reference for app mode -export function getDatatableSdkReference(): string { +// Helper to get the datatable SQL SDK reference (wmill.datatable()). +// Pass a language to get only that SDK; omit it to get both. +export function getDatatableSdkReference(language?: string): string { + if (language == null) { + return [ + prompts.DATATABLE_SDK_TYPESCRIPT, + prompts.DATATABLE_SDK_PYTHON + ].filter(Boolean).join('\n\n'); + } + if (TS_SDK_LANGUAGES.includes(language)) { + return prompts.DATATABLE_SDK_TYPESCRIPT; + } + if (PY_SDK_LANGUAGES.includes(language)) { + return prompts.DATATABLE_SDK_PYTHON; + } + // Unknown language: return both rather than nothing. return [ prompts.DATATABLE_SDK_TYPESCRIPT, prompts.DATATABLE_SDK_PYTHON diff --git a/system_prompts/generate.py b/system_prompts/generate.py index 0c03e47fb5..783dcc1ce3 100644 --- a/system_prompts/generate.py +++ b/system_prompts/generate.py @@ -2463,8 +2463,22 @@ export function getRawAppPrompt(): string { return prompts.RAW_APP_BASE; } -// Helper to get datatable SDK reference for app mode -export function getDatatableSdkReference(): string { +// Helper to get the datatable SQL SDK reference (wmill.datatable()). +// Pass a language to get only that SDK; omit it to get both. +export function getDatatableSdkReference(language?: string): string { + if (language == null) { + return [ + prompts.DATATABLE_SDK_TYPESCRIPT, + prompts.DATATABLE_SDK_PYTHON + ].filter(Boolean).join('\\n\\n'); + } + if (TS_SDK_LANGUAGES.includes(language)) { + return prompts.DATATABLE_SDK_TYPESCRIPT; + } + if (PY_SDK_LANGUAGES.includes(language)) { + return prompts.DATATABLE_SDK_PYTHON; + } + // Unknown language: return both rather than nothing. return [ prompts.DATATABLE_SDK_TYPESCRIPT, prompts.DATATABLE_SDK_PYTHON @@ -2501,7 +2515,7 @@ export declare function getScriptPrompt(language: string): string; export declare function getFlowPrompt(): string; export declare function getResourcePrompt(): string; export declare function getRawAppPrompt(): string; -export declare function getDatatableSdkReference(): string; +export declare function getDatatableSdkReference(language?: string): string; export declare function getWorkflowAsCodePrompt(language?: string): string; """ (OUTPUT_GENERATED_DIR / "index.d.ts").write_text(index_dts_content) From 1275487f028d4c74a9eeb18981ed05c225505be0 Mon Sep 17 00:00:00 2001 From: Guilhem Date: Mon, 1 Jun 2026 19:49:12 +0200 Subject: [PATCH 02/61] feat: refine ask-user-question chat display and keyboard nav (#9392) * feat: refine ask-user-question chat display and keyboard nav Co-Authored-By: Claude Opus 4.8 (1M context) * style: use text-accent for ask-user-question icon Co-Authored-By: Claude Opus 4.8 (1M context) * fix: focus active choice when clicking ask-user-question card Co-Authored-By: Claude Opus 4.8 (1M context) * feat: disable chat input while an ask-user-question is pending Co-Authored-By: Claude Opus 4.8 (1M context) * refactor: focus active choice on card click instead of pointerdown Preserves text selection on the question card; wired as a use: action so the non-interactive card needs no keyboard handler. Co-Authored-By: Claude Opus 4.8 (1M context) * refactor: extract isActiveUserQuestion shared predicate Co-Authored-By: Claude Opus 4.8 (1M context) * test: cover isActiveUserQuestion predicate Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Claude Opus 4.8 (1M context) --- .../copilot/chat/AIChatDisplay.svelte | 19 ++- .../chat/AskUserQuestionDisplay.svelte | 109 +++++++++++++----- .../copilot/chat/ToolExecutionDisplay.svelte | 10 +- .../components/copilot/chat/shared.test.ts | 63 ++++++++++ .../src/lib/components/copilot/chat/shared.ts | 16 +++ 5 files changed, 172 insertions(+), 45 deletions(-) diff --git a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte index bb97d61ee5..1252b999d7 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte +++ b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte @@ -23,7 +23,7 @@ import { fade } from 'svelte/transition' import Popover from '$lib/components/meltComponents/Popover.svelte' import DropdownV2 from '$lib/components/DropdownV2.svelte' - import { type DisplayMessage } from './shared' + import { isActiveUserQuestion, type DisplayMessage } from './shared' import type { ContextElement } from './context' import ChatQuickActions from './ChatQuickActions.svelte' import ProviderModelSelector from './ProviderModelSelector.svelte' @@ -271,18 +271,15 @@ const last = messages[messages.length - 1] if (!last || last.role !== 'tool') return false if (last.needsConfirmation && last.isLoading) return true - if ( - last.userQuestion && - last.isLoading && - !last.error && - !last.userQuestion.selectedChoice && - !last.userQuestion.canceled - ) { - return true - } + if (isActiveUserQuestion(last)) return true return false }) + // While the AI is waiting on an answer to an askUserQuestion, the only valid + // input is one of the choices (or the custom answer) in the question card — + // so disable the main chat input until the question is answered or canceled. + const hasActiveUserQuestion = $derived(isActiveUserQuestion(messages[messages.length - 1])) + // Get app context for display when in APP mode const appContext = $derived.by((): SelectedContext | undefined => { if (aiChatManager.mode !== AIMode.APP || !aiChatManager.appAiChatHelpers) { @@ -510,7 +507,7 @@ bind:this={aiChatInput} bind:selectedContext {availableContext} - {disabled} + disabled={disabled || hasActiveUserQuestion} isFirstMessage={messages.length === 0} />
import { onMount, tick } from 'svelte' - import { CircleHelp } from 'lucide-svelte' + import { CircleHelp, ArrowUp } from 'lucide-svelte' import Button from '$lib/components/common/button/Button.svelte' import TextInput from '$lib/components/text_input/TextInput.svelte' import { getAiChatManager } from './aiChatManagerContext' @@ -21,21 +21,31 @@ let { toolCallId, userQuestion }: Props = $props() let choiceButtons = $state<(HTMLButtonElement | undefined)[]>([]) + let customAnswerInput = $state() let customAnswer = $state('') let canSubmitCustomAnswer = $derived(customAnswer.trim().length > 0) - onMount(() => { - if (userQuestion.choices.length === 0) { - return - } + // The custom-answer input is the last stop in the roving cursor, after all + // choices, so arrow navigation can reach it from the keyboard. + const customAnswerIndex = $derived(userQuestion.choices.length) + const itemCount = $derived(userQuestion.choices.length + 1) + // The item currently under the keyboard cursor. Mirrored onto the matching + // choice's `selected` prop so the highlight comes from the Button itself. + let activeIndex = $state(0) + onMount(() => { void tick().then(() => { - focusChoice(0) + focusIndex(0) }) }) - function focusChoice(index: number) { - choiceButtons[index]?.focus() + function focusIndex(index: number) { + activeIndex = index + if (index === customAnswerIndex) { + customAnswerInput?.focus() + } else { + choiceButtons[index]?.focus() + } } function selectChoice(choice: string) { @@ -55,14 +65,14 @@ if (event.key === 'ArrowDown' || event.key === 'ArrowRight') { event.preventDefault() event.stopPropagation() - focusChoice((index + 1) % userQuestion.choices.length) + focusIndex((index + 1) % itemCount) return } if (event.key === 'ArrowUp' || event.key === 'ArrowLeft') { event.preventDefault() event.stopPropagation() - focusChoice((index - 1 + userQuestion.choices.length) % userQuestion.choices.length) + focusIndex((index - 1 + itemCount) % itemCount) return } @@ -73,24 +83,68 @@ } } + // Clicking the card's empty area must pull focus to the active item; otherwise + // focus stays on the chat's scroll container and arrow keys scroll the + // conversation instead of moving the selection. Wired as an action rather than + // a declarative on:click so the non-interactive card needs no keyboard handler, + // and on `click` rather than `pointerdown` so click-dragging to select text + // still works. + function focusActiveOnBackgroundClick(node: HTMLElement) { + function onClick(event: MouseEvent) { + const interactive = (event.target as HTMLElement | null)?.closest( + 'button, input, textarea, a, [contenteditable]' + ) + // A control inside the card (a choice, the answer input, the send button) + // manages its own focus — leave it alone. `closest` can also match the + // message-row button the chat wraps every message in, which is an ancestor + // of the card, so only bail out when the match is actually inside the card. + if (interactive && node.contains(interactive)) { + return + } + event.stopPropagation() + focusIndex(activeIndex) + } + node.addEventListener('click', onClick) + return { + destroy() { + node.removeEventListener('click', onClick) + } + } + } + function handleCustomAnswerKeydown(event: KeyboardEvent) { - if (event.key !== 'Enter') { + // Only ArrowUp/ArrowDown roam out of the input — Left/Right stay free for + // moving the text caret within the answer. + if (event.key === 'ArrowUp') { + event.preventDefault() + event.stopPropagation() + focusIndex((customAnswerIndex - 1 + itemCount) % itemCount) return } - event.preventDefault() - event.stopPropagation() - submitCustomAnswer() + if (event.key === 'ArrowDown') { + event.preventDefault() + event.stopPropagation() + focusIndex((customAnswerIndex + 1) % itemCount) + return + } + + if (event.key === 'Enter') { + event.preventDefault() + event.stopPropagation() + submitCustomAnswer() + } }
- -

+

{userQuestion.question}

@@ -100,37 +154,40 @@ {/each}
(activeIndex = customAnswerIndex) }} /> + />
diff --git a/frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte b/frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte index 48b9aab404..b6d01c6b38 100644 --- a/frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte +++ b/frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte @@ -4,7 +4,7 @@ import { getAiChatManager } from './aiChatManagerContext' const aiChatManager = getAiChatManager() - import type { ToolDisplayMessage } from './shared' + import { isActiveUserQuestion, type ToolDisplayMessage } from './shared' import { twMerge } from 'tailwind-merge' import ToolContentDisplay from './ToolContentDisplay.svelte' import ToolMessageActions from './ToolMessageActions.svelte' @@ -41,13 +41,7 @@ ) const activeUserQuestion = $derived( - message.userQuestion && - message.isLoading && - !message.error && - !message.userQuestion.selectedChoice && - !message.userQuestion.canceled - ? message.userQuestion - : undefined + isActiveUserQuestion(message) ? message.userQuestion : undefined ) diff --git a/frontend/src/lib/components/copilot/chat/shared.test.ts b/frontend/src/lib/components/copilot/chat/shared.test.ts index 69b868ee50..56b6bc4d2b 100644 --- a/frontend/src/lib/components/copilot/chat/shared.test.ts +++ b/frontend/src/lib/components/copilot/chat/shared.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it, vi } from 'vitest' import { z } from 'zod' +import type { DisplayMessage, ToolDisplayMessage } from './shared' vi.mock('monaco-editor', () => ({ editor: {} @@ -618,3 +619,65 @@ describe('processToolCall', () => { ) }) }) + +describe('isActiveUserQuestion', () => { + function toolMessage(overrides: Partial = {}): ToolDisplayMessage { + return { + role: 'tool', + tool_call_id: 'call_q', + content: 'asking a question', + isLoading: true, + userQuestion: { question: 'Pick one', choices: ['a', 'b'] }, + ...overrides + } + } + + it('is true for a loading tool message with an unanswered question', async () => { + const { isActiveUserQuestion } = await import('./shared') + expect(isActiveUserQuestion(toolMessage())).toBe(true) + }) + + it('is false once a choice has been selected', async () => { + const { isActiveUserQuestion } = await import('./shared') + expect( + isActiveUserQuestion( + toolMessage({ + userQuestion: { question: 'Pick one', choices: ['a', 'b'], selectedChoice: 'a' } + }) + ) + ).toBe(false) + }) + + it('is false when the question was canceled', async () => { + const { isActiveUserQuestion } = await import('./shared') + expect( + isActiveUserQuestion( + toolMessage({ userQuestion: { question: 'Pick one', choices: ['a', 'b'], canceled: true } }) + ) + ).toBe(false) + }) + + it('is false when the tool errored', async () => { + const { isActiveUserQuestion } = await import('./shared') + expect(isActiveUserQuestion(toolMessage({ error: 'boom' }))).toBe(false) + }) + + it('is false when the tool is no longer loading', async () => { + const { isActiveUserQuestion } = await import('./shared') + expect(isActiveUserQuestion(toolMessage({ isLoading: false }))).toBe(false) + }) + + it('is false for a tool message without a question', async () => { + const { isActiveUserQuestion } = await import('./shared') + expect(isActiveUserQuestion(toolMessage({ userQuestion: undefined }))).toBe(false) + }) + + it('is false for non-tool messages and undefined', async () => { + const { isActiveUserQuestion } = await import('./shared') + const userMessage: DisplayMessage = { role: 'user', index: 0, content: 'hi' } + const assistantMessage: DisplayMessage = { role: 'assistant', content: 'hi' } + expect(isActiveUserQuestion(undefined)).toBe(false) + expect(isActiveUserQuestion(userMessage)).toBe(false) + expect(isActiveUserQuestion(assistantMessage)).toBe(false) + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/shared.ts b/frontend/src/lib/components/copilot/chat/shared.ts index 97089e8db0..7764b85f2f 100644 --- a/frontend/src/lib/components/copilot/chat/shared.ts +++ b/frontend/src/lib/components/copilot/chat/shared.ts @@ -514,6 +514,22 @@ export type AssistantDisplayMessage = BaseDisplayMessage & { export type DisplayMessage = UserDisplayMessage | ToolDisplayMessage | AssistantDisplayMessage +// A tool message whose askUserQuestion is still awaiting an answer: the AI loop +// is paused on the user. Drives the question card's interactivity, the +// "waiting for user" indicator, and disabling the main chat input — keep those +// in sync by going through this single predicate. +export function isActiveUserQuestion(message: DisplayMessage | undefined): boolean { + return Boolean( + message && + message.role === 'tool' && + message.userQuestion && + message.isLoading && + !message.error && + !message.userQuestion.selectedChoice && + !message.userQuestion.canceled + ) +} + async function callTool({ tools, functionName, From ba0e4c8280d91896c082c3bdd375c1c8b49117d6 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Mon, 1 Jun 2026 19:49:33 +0200 Subject: [PATCH 03/61] oauth: add salesforce provider (#9380) * oauth: add salesforce provider Register Salesforce OAuth (Authorization Code) for Windmill resource connect. Production uses login.salesforce.com; the sandbox block points at test.salesforce.com (URL overrides only; scopes inherited) per #9358, so a single canonical `salesforce` resource type covers both with separate `salesforce_sandbox` instance credentials. Paired with the hub integration: windmill-labs/windmill-integrations#131. The Salesforce icon already exists in the frontend (SalesforceIcon.svelte). Co-Authored-By: Claude Opus 4.8 (1M context) * Fix JSON syntax error in oauth_connect.json * fix: add salesforce production tile to OAuth settings dropdown --------- Co-authored-by: Claude Opus 4.8 (1M context) --- backend/oauth_connect.json | 13 +++++++++++++ frontend/src/lib/components/AuthSettings.svelte | 3 ++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/backend/oauth_connect.json b/backend/oauth_connect.json index d18c8c8d24..36f565fc2b 100644 --- a/backend/oauth_connect.json +++ b/backend/oauth_connect.json @@ -181,5 +181,18 @@ "auth_url": "https://account-d.docusign.com/oauth/auth", "token_url": "https://account-d.docusign.com/oauth/token" } + }, + "salesforce": { + "auth_url": "https://login.salesforce.com/services/oauth2/authorize", + "token_url": "https://login.salesforce.com/services/oauth2/token", + "scopes": [ + "api", + "refresh_token", + "offline_access" + ], + "sandbox": { + "auth_url": "https://test.salesforce.com/services/oauth2/authorize", + "token_url": "https://test.salesforce.com/services/oauth2/token" + } } } diff --git a/frontend/src/lib/components/AuthSettings.svelte b/frontend/src/lib/components/AuthSettings.svelte index 487f31618a..734c6a1fe3 100644 --- a/frontend/src/lib/components/AuthSettings.svelte +++ b/frontend/src/lib/components/AuthSettings.svelte @@ -84,7 +84,8 @@ 'zoho', 'xero', 'apify', - 'docusign' + 'docusign', + 'salesforce' ] // Providers whose registry entry (`backend/oauth_connect.json`) carries a // `sandbox` URL block. Each one gets a sibling `_sandbox` dropdown From de76668c10c04abe8771a8ca7bba7b2259819a1c Mon Sep 17 00:00:00 2001 From: Guilhem Date: Tue, 2 Jun 2026 01:29:19 +0200 Subject: [PATCH 04/61] fix(frontend): align Monaco editor font size with text-xs (#9161) * fix(frontend): align Monaco editor font size with text-xs across viewports * fix(frontend): make placeholder lineHeight reactive to fontSize * fix(frontend): align GraphQL schema viewer font size with text-xs The read-only GraphQL schema viewer was the lone Monaco instance still inheriting Monaco's 14px default. Wire it through editorFontSize like the other editors so it stays in sync with text-xs across viewports. Co-Authored-By: Claude Opus 4.7 --------- Co-authored-by: Claude Opus 4.7 --- frontend/src/lib/components/DiffEditor.svelte | 9 +++++++ frontend/src/lib/components/Editor.svelte | 10 +++++++- .../components/FakeMonacoPlaceHolder.svelte | 8 +++--- .../lib/components/GraphqlSchemaViewer.svelte | 17 +++++++++---- .../src/lib/components/SimpleEditor.svelte | 10 +++++++- .../src/lib/components/TemplateEditor.svelte | 16 +++++++++--- frontend/src/lib/editorFontSize.svelte.ts | 25 +++++++++++++++++++ 7 files changed, 82 insertions(+), 13 deletions(-) create mode 100644 frontend/src/lib/editorFontSize.svelte.ts diff --git a/frontend/src/lib/components/DiffEditor.svelte b/frontend/src/lib/components/DiffEditor.svelte index 36d6148054..59bc8a33a5 100644 --- a/frontend/src/lib/components/DiffEditor.svelte +++ b/frontend/src/lib/components/DiffEditor.svelte @@ -8,6 +8,7 @@ import { editor as meditor, KeyMod, KeyCode } from 'monaco-editor' import { initializeVscode } from './vscode' + import { editorFontSize } from '$lib/editorFontSize.svelte' import { registerWebviewPaste } from '$lib/editorUtils' import EditorTheme from './EditorTheme.svelte' import Button from '$lib/components/common/button/Button.svelte' @@ -70,6 +71,7 @@ scrollBeyondLastLine: false, lineDecorationsWidth: 15, lineNumbersMinChars: 2, + fontSize: editorFontSize.regular, scrollbar: { alwaysConsumeMouseWheel: false } }) @@ -175,6 +177,13 @@ } }) + $effect(() => { + const fontSize = editorFontSize.regular + if (diffEditor) { + diffEditor.updateOptions({ fontSize }) + } + }) + $effect(() => { if (!diffEditor) { return diff --git a/frontend/src/lib/components/Editor.svelte b/frontend/src/lib/components/Editor.svelte index 14036a0ab7..698e799aad 100644 --- a/frontend/src/lib/components/Editor.svelte +++ b/frontend/src/lib/components/Editor.svelte @@ -40,6 +40,7 @@ } from '$lib/stores' import { editorConfig, registerWebviewPaste, updateOptions } from '$lib/editorUtils' + import { editorFontSize } from '$lib/editorFontSize.svelte' import { createHash as randomHash } from '$lib/editorLangUtils' import { workspaceStore } from '$lib/stores' import { @@ -1380,7 +1381,7 @@ $relativeLineNumbers ), model, - fontSize: !small ? 13.5 : 12, + fontSize: small ? editorFontSize.small : editorFontSize.regular, lineNumbersMinChars, // overflowWidgetsDomNode: widgets, tabSize: lang == 'python' ? 4 : 2, @@ -1755,6 +1756,13 @@ let aiChatInlineWidget: AIChatInlineWidget | null = $state(null) + $effect(() => { + const fontSize = small ? editorFontSize.small : editorFontSize.regular + if (editor) { + editor.updateOptions({ fontSize }) + } + }) + let loadTimeout: number | undefined = undefined onMount(async () => { if (BROWSER) { diff --git a/frontend/src/lib/components/FakeMonacoPlaceHolder.svelte b/frontend/src/lib/components/FakeMonacoPlaceHolder.svelte index c7f068a83f..3f486494a3 100644 --- a/frontend/src/lib/components/FakeMonacoPlaceHolder.svelte +++ b/frontend/src/lib/components/FakeMonacoPlaceHolder.svelte @@ -1,8 +1,8 @@ @@ -696,7 +706,7 @@ class={twMerge(inputBorderClass({ forceFocus: isFocus }), 'rounded-md overflow-auto pl-2', clazz)} > {#if !editor} - + {/if}
{ + isLargeViewport = e.matches + }) +} + +export const editorFontSize = { + get regular(): number { + return isLargeViewport ? 13.5 : 12 + }, + get small(): number { + return isLargeViewport ? 12 : 11 + } +} From 2e1445616a412c5112ad2247b4087c7ddc218845 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 2 Jun 2026 01:37:35 +0200 Subject: [PATCH 05/61] feat: handle CTRL_BREAK_EVENT for graceful shutdown on Windows (#9400) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On Windows, shutdown_signal only registered ctrl_c() (CTRL_C_EVENT). CTRL_BREAK_EVENT — the default kill signal sent by Nomad's raw_exec driver on Windows — had no handler, so the worker terminated immediately without graceful shutdown, interrupting running jobs. Add a ctrl_break() helper (mirroring the Unix terminate() helper) and register it as an additional branch in both Windows tokio::select! blocks in shutdown_signal. Fixes WIN-2003 Co-authored-by: Claude Opus 4.8 (1M context) --- backend/windmill-common/src/lib.rs | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index 95db60cc3b..7cbb14ec9a 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -282,6 +282,12 @@ pub async fn shutdown_signal( Ok(()) } + #[cfg(windows)] + async fn ctrl_break() -> std::io::Result<()> { + tokio::signal::windows::ctrl_break()?.recv().await; + Ok(()) + } + #[cfg(any(target_os = "linux", target_os = "macos"))] tokio::select! { _ = terminate() => { @@ -297,7 +303,13 @@ pub async fn shutdown_signal( #[cfg(not(any(target_os = "linux", target_os = "macos")))] tokio::select! { - _ = tokio::signal::ctrl_c() => {}, + _ = tokio::signal::ctrl_c() => { + tracing::info!("shutdown monitor received ctrl-c"); + }, + #[cfg(windows)] + _ = ctrl_break() => { + tracing::info!("shutdown monitor received ctrl-break"); + }, _ = rx.recv() => { tracing::info!("shutdown monitor received killpill"); }, @@ -319,6 +331,10 @@ pub async fn shutdown_signal( _ = tokio::signal::ctrl_c() => { tracing::error!("2nd shutdown monitor received ctrl-c") }, + #[cfg(windows)] + _ = ctrl_break() => { + tracing::error!("2nd shutdown monitor received ctrl-break") + }, } tracing::info!("Second terminate signal received, forcefully exiting"); From e8ad53dae92597f5a1a8b76f38a7d8c24f578a47 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 2 Jun 2026 08:43:03 +0200 Subject: [PATCH 06/61] fix: resolve username rename failing on apps with runnable deps (#9401) The instance username-conflict resolver rewrote workspace_runnable_dependencies.app_path to the new user path before the app row itself was renamed, violating fk_workspace_runnable_dependencies_app_path. That FK is ON UPDATE CASCADE, so renaming the app already propagates the new path; the manual rewrite was redundant and mis-ordered. Any user owning an app under u// with a tracked runnable dependency hit HTTP 500 and could not have their username conflict resolved. Co-authored-by: Claude Opus 4.8 --- ...a6bf2dc0746d9f3a1bea104123188fe2921bc886.json | 16 ---------------- backend/windmill-api/src/users.rs | 14 +++++++------- 2 files changed, 7 insertions(+), 23 deletions(-) delete mode 100644 backend/.sqlx/query-f699cc3644aeb35a0588bbb3a6bf2dc0746d9f3a1bea104123188fe2921bc886.json diff --git a/backend/.sqlx/query-f699cc3644aeb35a0588bbb3a6bf2dc0746d9f3a1bea104123188fe2921bc886.json b/backend/.sqlx/query-f699cc3644aeb35a0588bbb3a6bf2dc0746d9f3a1bea104123188fe2921bc886.json deleted file mode 100644 index d352d3d69d..0000000000 --- a/backend/.sqlx/query-f699cc3644aeb35a0588bbb3a6bf2dc0746d9f3a1bea104123188fe2921bc886.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE workspace_runnable_dependencies SET app_path = REGEXP_REPLACE(app_path,'u/' || $2 || '/(.*)','u/' || $1 || '/\\1') WHERE app_path LIKE ('u/' || $2 || '/%') AND workspace_id = $3", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "f699cc3644aeb35a0588bbb3a6bf2dc0746d9f3a1bea104123188fe2921bc886" -} diff --git a/backend/windmill-api/src/users.rs b/backend/windmill-api/src/users.rs index 58871866b1..d53998355b 100644 --- a/backend/windmill-api/src/users.rs +++ b/backend/windmill-api/src/users.rs @@ -484,13 +484,13 @@ async fn update_username_in_workpsace<'c>( ).execute(&mut **tx) .await?; - sqlx::query!( - r#"UPDATE workspace_runnable_dependencies SET app_path = REGEXP_REPLACE(app_path,'u/' || $2 || '/(.*)','u/' || $1 || '/\1') WHERE app_path LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#, - new_username, - old_username, - w_id - ).execute(&mut **tx) - .await?; + // NB: workspace_runnable_dependencies.app_path is intentionally NOT rewritten here. + // Its FK to app(path, workspace_id) is ON UPDATE CASCADE, so the `UPDATE app SET path` + // below propagates the new path automatically. Rewriting it manually here (before the + // app row is renamed) points the row at a not-yet-existing app path and violates + // fk_workspace_runnable_dependencies_app_path. (flow_path above DOES need the manual + // rewrite because flows are migrated via INSERT-new + DELETE-old, not UPDATE flow.path, + // so the cascade never fires for them.) sqlx::query!( r#"UPDATE workspace_runnable_dependencies SET runnable_path = REGEXP_REPLACE(runnable_path,'u/' || $2 || '/(.*)','u/' || $1 || '/\1') WHERE runnable_path LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#, From d71d553ba46ba63ac57d4ddb5a5bfb04e5aeaacb Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 2 Jun 2026 09:10:53 +0200 Subject: [PATCH 07/61] Windows build broken by #[cfg] on tokio::select! branch (#9404) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #9400 (WIN-2003) added the ctrl_break() handler as a `#[cfg(windows)]` branch inside the two Windows-path `tokio::select!` blocks in shutdown_signal. tokio's `select!` macro does not accept `#[cfg(...)]` attributes on individual branches, so windmill-common fails to compile on Windows ("no rules expected this token in macro call"). This slipped through CI because the only job that builds the backend on Windows is cli-tests.yml's `test-windows`, which triggers only on `cli/**` changes — #9400 was backend-only. Fix: define `ctrl_break()` for the whole `not(any(linux, macos))` scope instead of just `windows`. On Windows it awaits the real CTRL_BREAK signal; on other non-unix targets it is a never-resolving future, so the branch is inert there. The select! branches become plain (no per-branch `#[cfg]`), which the macro accepts. Verified: the `#[cfg]`-on-branch form reproduces the exact macro error against tokio 1.46.1, and the fixed form compiles clean. Fixes WIN-2003 (Windows build regression) Co-authored-by: Claude Opus 4.8 (1M context) --- backend/windmill-common/src/lib.rs | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index 7cbb14ec9a..c12d242737 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -282,10 +282,22 @@ pub async fn shutdown_signal( Ok(()) } - #[cfg(windows)] + // Defined for the whole non-unix scope (not just windows) so it can be a + // plain `tokio::select!` branch: that macro does not accept `#[cfg(...)]` + // attributes on individual branches. On non-windows non-unix targets the + // future never resolves, so the branch is effectively inert there. + #[cfg(not(any(target_os = "linux", target_os = "macos")))] async fn ctrl_break() -> std::io::Result<()> { - tokio::signal::windows::ctrl_break()?.recv().await; - Ok(()) + #[cfg(windows)] + { + tokio::signal::windows::ctrl_break()?.recv().await; + Ok(()) + } + #[cfg(not(windows))] + { + std::future::pending::<()>().await; + Ok(()) + } } #[cfg(any(target_os = "linux", target_os = "macos"))] @@ -306,7 +318,6 @@ pub async fn shutdown_signal( _ = tokio::signal::ctrl_c() => { tracing::info!("shutdown monitor received ctrl-c"); }, - #[cfg(windows)] _ = ctrl_break() => { tracing::info!("shutdown monitor received ctrl-break"); }, @@ -331,7 +342,6 @@ pub async fn shutdown_signal( _ = tokio::signal::ctrl_c() => { tracing::error!("2nd shutdown monitor received ctrl-c") }, - #[cfg(windows)] _ = ctrl_break() => { tracing::error!("2nd shutdown monitor received ctrl-break") }, From e356bb1f5df92eca3fbb0ca2114b9f4c32d4c496 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 2 Jun 2026 09:11:29 +0200 Subject: [PATCH 08/61] fix(cli): make encryption key push non-interactive-safe + add --skip-reencrypt-on-key-change (#9402) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When encryption_key.yaml changes and is pushed via `wmill sync push`, pushWorkspaceKey prompted interactively to confirm re-encrypting the remote secrets with the new key. That prompt ignored `--yes` and had no TTY guard, so a CI/non-interactive push that included the key would block (or behave undefinedly) on the prompt. Thread a key-push options object (non-interactive flag + explicit re-encryption choice) through pushObj into pushWorkspaceKey: - Non-interactive (`--yes` or no TTY) and no explicit choice: skip the prompt and default to re-encrypting all remote secrets with the new key (matches the interactive default), preserving their plaintext values. - New `--skip-reencrypt-on-key-change` flag (and the WMILL_NO_REENCRYPT_ON_KEY_CHANGE=true env var for CI) opt out of re-encryption — only safe when the remote ciphertexts are already encrypted with the new key (e.g. workspace/instance migration). - Interactive behavior (TTY, no `--yes`) is unchanged. Regenerates system_prompts for the new option and adds unit tests for the no-op, re-encrypt-by-default, flag-skip, and env-skip paths. Fixes WIN-2005 Co-authored-by: Claude Opus 4.8 (1M context) --- cli/src/commands/sync/sync.ts | 12 +++ cli/src/core/conf.ts | 1 + cli/src/core/settings.ts | 55 +++++++++-- cli/src/guidance/skills.gen.ts | 1 + cli/src/types.ts | 10 +- cli/test/push_workspace_key_unit.test.ts | 93 +++++++++++++++++++ .../auto-generated/cli/cli-commands.md | 1 + system_prompts/auto-generated/prompts.ts | 1 + .../skills/cli-commands/SKILL.md | 1 + 9 files changed, 166 insertions(+), 9 deletions(-) create mode 100644 cli/test/push_workspace_key_unit.test.ts diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index 2632f6c80c..dda0f41035 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -4041,6 +4041,10 @@ export async function push( originalWorkspaceSpecificPath, permissionedAsContext, isWsSpecific ? true : undefined, + { + noninteractive: (opts.yes ?? false) || !process.stdin.isTTY, + skipReencrypt: opts.skipReencryptOnKeyChange, + }, ); if (stateTarget) { @@ -4126,6 +4130,10 @@ export async function push( localFilePath, // Pass the actual local file path permissionedAsContext, isAddedWsSpecific ? true : undefined, + { + noninteractive: (opts.yes ?? false) || !process.stdin.isTTY, + skipReencrypt: opts.skipReencryptOnKeyChange, + }, ); if (stateTarget) { @@ -4682,6 +4690,10 @@ const command = new Command() .option("--include-groups", "Include syncing groups") .option("--include-settings", "Include syncing workspace settings") .option("--include-key", "Include workspace encryption key") + .option( + "--skip-reencrypt-on-key-change", + "When the pushed encryption key differs from the remote, do NOT re-encrypt existing remote secrets. Only safe if they are already encrypted with the new key (e.g. workspace/instance migration). Default is to re-encrypt.", + ) .option("--skip-branch-validation", "Skip git branch validation and prompts") .option("--json-output", "Output results in JSON format") .option( diff --git a/cli/src/core/conf.ts b/cli/src/core/conf.ts index 5656cfe9d4..c4fda73b1b 100644 --- a/cli/src/core/conf.ts +++ b/cli/src/core/conf.ts @@ -88,6 +88,7 @@ export interface SyncOptions { includeGroups?: boolean; includeSettings?: boolean; includeKey?: boolean; + skipReencryptOnKeyChange?: boolean; skipBranchValidation?: boolean; message?: string; includes?: string[]; diff --git a/cli/src/core/settings.ts b/cli/src/core/settings.ts index 4b86cb3469..e4b618180f 100644 --- a/cli/src/core/settings.ts +++ b/cli/src/core/settings.ts @@ -445,11 +445,23 @@ export async function pushWorkspaceSettings( } } +export interface PushWorkspaceKeyOptions { + // True when no prompt may be shown (e.g. `--yes` was passed or stdin is not a + // TTY). In that case the re-encryption decision is taken from `skipReencrypt` + // / the WMILL_NO_REENCRYPT_ON_KEY_CHANGE env var instead of an interactive + // confirmation. + noninteractive?: boolean; + // Explicit re-encryption decision from `--skip-reencrypt-on-key-change`. + // When set it takes precedence over the prompt and the env var. + skipReencrypt?: boolean; +} + export async function pushWorkspaceKey( workspace: string, _path: string, key: string | undefined, - localKey: string + localKey: string, + opts?: PushWorkspaceKeyOptions ) { try { key = await wmill @@ -461,17 +473,46 @@ export async function pushWorkspaceKey( throw new Error(`Failed to get workspace encryption key: ${err}`); } if (localKey && key !== localKey) { - const confirm = await Confirm.prompt({ - message: - "The local workspace encryption key does not match the remote. Do you want to reencrypt all your secrets on the remote with the new key?\nSay 'no' if your local secrets are already encrypted with the new key (e.g. workspace/instance migration)\nOtherwise, say 'yes' and pull the secrets after the reencryption.\n", - default: true, - }); + // Changing the key on the remote means the existing ciphertexts (encrypted + // with the old key) become unreadable unless they are re-encrypted. By + // default we ask the backend to re-encrypt every secret variable with the + // new key, which preserves their plaintext values. The only reason to skip + // re-encryption is when the stored ciphertexts are *already* encrypted with + // the new key (e.g. a workspace/instance migration). + let reencrypt: boolean; + // Explicit choice via `--skip-reencrypt-on-key-change` or the env var wins + // over everything, regardless of interactivity. + const explicitSkip = + opts?.skipReencrypt || + (process.env.WMILL_NO_REENCRYPT_ON_KEY_CHANGE ?? "").toLowerCase() === + "true"; + if (explicitSkip) { + reencrypt = false; + log.info( + "Workspace encryption key changed; leaving remote ciphertexts untouched (skip re-encryption requested)." + ); + } else if (opts?.noninteractive) { + // No TTY (or --yes) and no explicit skip: we can't prompt, so default to + // re-encrypting (matches the interactive default) to preserve secret + // values. Pass --skip-reencrypt-on-key-change (or set + // WMILL_NO_REENCRYPT_ON_KEY_CHANGE=true) to opt out. + reencrypt = true; + log.info( + "Workspace encryption key changed; re-encrypting all remote secrets with the new key (non-interactive)." + ); + } else { + reencrypt = await Confirm.prompt({ + message: + "The local workspace encryption key does not match the remote. Do you want to reencrypt all your secrets on the remote with the new key?\nSay 'no' if your local secrets are already encrypted with the new key (e.g. workspace/instance migration)\nOtherwise, say 'yes' and pull the secrets after the reencryption.\n", + default: true, + }); + } log.debug(`Updating workspace encryption key...`); await wmill.setWorkspaceEncryptionKey({ workspace, requestBody: { new_key: localKey, - skip_reencrypt: !confirm, + skip_reencrypt: !reencrypt, }, }); } else { diff --git a/cli/src/guidance/skills.gen.ts b/cli/src/guidance/skills.gen.ts index 7dd25dc705..8b6f21053d 100644 --- a/cli/src/guidance/skills.gen.ts +++ b/cli/src/guidance/skills.gen.ts @@ -6608,6 +6608,7 @@ sync local with a remote workspaces or the opposite (push or pull) - \`--include-groups\` - Include syncing groups - \`--include-settings\` - Include syncing workspace settings - \`--include-key\` - Include workspace encryption key + - \`--skip-reencrypt-on-key-change\` - When the pushed encryption key differs from the remote, do NOT re-encrypt existing remote secrets. Only safe if they are already encrypted with the new key (e.g. workspace/instance migration). Default is to re-encrypt. - \`--skip-branch-validation\` - Skip git branch validation and prompts - \`--json-output\` - Output results in JSON format - \`-i --includes \` - Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string) diff --git a/cli/src/types.ts b/cli/src/types.ts index 75bad535c8..5de6cea48b 100644 --- a/cli/src/types.ts +++ b/cli/src/types.ts @@ -18,7 +18,11 @@ import { pushSchedule } from "./commands/schedule/schedule.ts"; import { pushWorkspaceUser } from "./commands/user/user.ts"; import { pushGroup } from "./commands/user/user.ts"; import { pushWorkspaceDependencies } from "./commands/dependencies/dependencies.ts"; -import { pushWorkspaceSettings, pushWorkspaceKey } from "./core/settings.ts"; +import { + pushWorkspaceSettings, + pushWorkspaceKey, + PushWorkspaceKeyOptions, +} from "./core/settings.ts"; import { pushTrigger, pushNativeTrigger } from "./commands/trigger/trigger.ts"; import { pushRawApp } from "./commands/app/raw_apps.ts"; import type { PermissionedAsContext } from "./core/permissioned_as.ts"; @@ -179,6 +183,7 @@ function redactString(s: string): string { * @param alreadySynced - Array to track already synced items * @param message - Optional commit/update message * @param originalLocalPath - The original local file path (used for branch-specific resource file resolution) + * @param keyPushOpts - Options for the encryption_key push: non-interactive flag and explicit re-encryption choice */ export async function pushObj( workspace: string, @@ -191,6 +196,7 @@ export async function pushObj( originalLocalPath?: string, permissionedAsContext?: PermissionedAsContext, wsSpecific?: boolean, + keyPushOpts?: PushWorkspaceKeyOptions, ) { const typeEnding = getTypeStrFromPath(p); @@ -256,7 +262,7 @@ export async function pushObj( } else if (typeEnding === "settings") { await pushWorkspaceSettings(workspace, p, befObj, newObj); } else if (typeEnding === "encryption_key") { - await pushWorkspaceKey(workspace, p, befObj, newObj); + await pushWorkspaceKey(workspace, p, befObj, newObj, keyPushOpts); } else { throw new Error( `The item ${p} has an unrecognized type ending ${typeEnding}` diff --git a/cli/test/push_workspace_key_unit.test.ts b/cli/test/push_workspace_key_unit.test.ts new file mode 100644 index 0000000000..6d8950cbec --- /dev/null +++ b/cli/test/push_workspace_key_unit.test.ts @@ -0,0 +1,93 @@ +/** + * Unit tests for pushWorkspaceKey in settings.ts. + * + * Covers WIN-2005: changing the encryption key in encryption_key.yaml and + * pushing it must (by default) re-encrypt the remote secrets with the new key. + * + * Verifies that: + * - an unchanged key is a no-op (no setWorkspaceEncryptionKey call) + * - a changed key in non-interactive mode re-encrypts by default + * (skip_reencrypt = false), so secret plaintext values are preserved + * - the --skip-reencrypt-on-key-change flag keeps the remote ciphertexts + * untouched (skip_reencrypt = true) + * - WMILL_NO_REENCRYPT_ON_KEY_CHANGE=true does the same via env var + */ + +import { expect, test, describe, beforeEach, afterEach, mock } from "bun:test"; + +// Track calls to mocked wmill functions +let remoteKey = ""; +let setEncryptionKeyCalls: { + workspace: string; + requestBody: { new_key: string; skip_reencrypt?: boolean }; +}[] = []; + +// Mock the wmill module before importing settings.ts +mock.module("../gen/services.gen.ts", () => ({ + getWorkspaceEncryptionKey: async (_args: { workspace: string }) => ({ + key: remoteKey, + }), + setWorkspaceEncryptionKey: async (args: { + workspace: string; + requestBody: { new_key: string; skip_reencrypt?: boolean }; + }) => { + setEncryptionKeyCalls.push(args); + }, +})); + +import { pushWorkspaceKey } from "../src/core/settings.ts"; + +describe("pushWorkspaceKey", () => { + const ws = "test-workspace"; + + beforeEach(() => { + remoteKey = ""; + setEncryptionKeyCalls = []; + delete process.env.WMILL_NO_REENCRYPT_ON_KEY_CHANGE; + }); + + afterEach(() => { + delete process.env.WMILL_NO_REENCRYPT_ON_KEY_CHANGE; + }); + + test("no-op when local key matches the remote key", async () => { + remoteKey = "samekey"; + await pushWorkspaceKey(ws, "encryption_key", undefined, "samekey", { + noninteractive: true, + }); + expect(setEncryptionKeyCalls.length).toBe(0); + }); + + test("changed key re-encrypts by default in non-interactive mode", async () => { + remoteKey = "oldkey"; + await pushWorkspaceKey(ws, "encryption_key", undefined, "newkey", { + noninteractive: true, + }); + expect(setEncryptionKeyCalls.length).toBe(1); + expect(setEncryptionKeyCalls[0].requestBody.new_key).toBe("newkey"); + // skip_reencrypt false => backend re-encrypts existing secrets with new key + expect(setEncryptionKeyCalls[0].requestBody.skip_reencrypt).toBe(false); + }); + + test("--skip-reencrypt-on-key-change skips re-encryption", async () => { + remoteKey = "oldkey"; + await pushWorkspaceKey(ws, "encryption_key", undefined, "newkey", { + noninteractive: true, + skipReencrypt: true, + }); + expect(setEncryptionKeyCalls.length).toBe(1); + expect(setEncryptionKeyCalls[0].requestBody.new_key).toBe("newkey"); + expect(setEncryptionKeyCalls[0].requestBody.skip_reencrypt).toBe(true); + }); + + test("WMILL_NO_REENCRYPT_ON_KEY_CHANGE=true skips re-encryption non-interactively", async () => { + remoteKey = "oldkey"; + process.env.WMILL_NO_REENCRYPT_ON_KEY_CHANGE = "true"; + await pushWorkspaceKey(ws, "encryption_key", undefined, "newkey", { + noninteractive: true, + }); + expect(setEncryptionKeyCalls.length).toBe(1); + expect(setEncryptionKeyCalls[0].requestBody.new_key).toBe("newkey"); + expect(setEncryptionKeyCalls[0].requestBody.skip_reencrypt).toBe(true); + }); +}); diff --git a/system_prompts/auto-generated/cli/cli-commands.md b/system_prompts/auto-generated/cli/cli-commands.md index b02aabefe0..662bb58a53 100644 --- a/system_prompts/auto-generated/cli/cli-commands.md +++ b/system_prompts/auto-generated/cli/cli-commands.md @@ -583,6 +583,7 @@ sync local with a remote workspaces or the opposite (push or pull) - `--include-groups` - Include syncing groups - `--include-settings` - Include syncing workspace settings - `--include-key` - Include workspace encryption key + - `--skip-reencrypt-on-key-change` - When the pushed encryption key differs from the remote, do NOT re-encrypt existing remote secrets. Only safe if they are already encrypted with the new key (e.g. workspace/instance migration). Default is to re-encrypt. - `--skip-branch-validation` - Skip git branch validation and prompts - `--json-output` - Output results in JSON format - `-i --includes ` - Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string) diff --git a/system_prompts/auto-generated/prompts.ts b/system_prompts/auto-generated/prompts.ts index 19e69543e4..b0341495d3 100644 --- a/system_prompts/auto-generated/prompts.ts +++ b/system_prompts/auto-generated/prompts.ts @@ -3133,6 +3133,7 @@ sync local with a remote workspaces or the opposite (push or pull) - \`--include-groups\` - Include syncing groups - \`--include-settings\` - Include syncing workspace settings - \`--include-key\` - Include workspace encryption key + - \`--skip-reencrypt-on-key-change\` - When the pushed encryption key differs from the remote, do NOT re-encrypt existing remote secrets. Only safe if they are already encrypted with the new key (e.g. workspace/instance migration). Default is to re-encrypt. - \`--skip-branch-validation\` - Skip git branch validation and prompts - \`--json-output\` - Output results in JSON format - \`-i --includes \` - Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string) diff --git a/system_prompts/auto-generated/skills/cli-commands/SKILL.md b/system_prompts/auto-generated/skills/cli-commands/SKILL.md index 4b3e72a8b7..9eade530c7 100644 --- a/system_prompts/auto-generated/skills/cli-commands/SKILL.md +++ b/system_prompts/auto-generated/skills/cli-commands/SKILL.md @@ -588,6 +588,7 @@ sync local with a remote workspaces or the opposite (push or pull) - `--include-groups` - Include syncing groups - `--include-settings` - Include syncing workspace settings - `--include-key` - Include workspace encryption key + - `--skip-reencrypt-on-key-change` - When the pushed encryption key differs from the remote, do NOT re-encrypt existing remote secrets. Only safe if they are already encrypted with the new key (e.g. workspace/instance migration). Default is to re-encrypt. - `--skip-branch-validation` - Skip git branch validation and prompts - `--json-output` - Output results in JSON format - `-i --includes ` - Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string) From 30057445f9e732e7fc3f3d460aa63c76cc728a03 Mon Sep 17 00:00:00 2001 From: Guilhem Date: Tue, 2 Jun 2026 09:16:55 +0200 Subject: [PATCH 09/61] avoid crypto.randomUUID in WorkspaceItemDrillPicker (#9405) Co-authored-by: Claude Opus 4.8 (1M context) --- frontend/src/lib/components/WorkspaceItemDrillPicker.svelte | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frontend/src/lib/components/WorkspaceItemDrillPicker.svelte b/frontend/src/lib/components/WorkspaceItemDrillPicker.svelte index 9136f52205..dda620dad5 100644 --- a/frontend/src/lib/components/WorkspaceItemDrillPicker.svelte +++ b/frontend/src/lib/components/WorkspaceItemDrillPicker.svelte @@ -20,6 +20,7 @@ Clicking a row drills *down*; the chevron-left in the header walks one level import WorkspaceItemRow from '$lib/components/WorkspaceItemRow.svelte' import SearchItems from '$lib/components/SearchItems.svelte' import { onMount, untrack } from 'svelte' + import { generateRandomString } from '$lib/utils' import { dirKey, getCachedItems, @@ -65,7 +66,7 @@ Clicking a row drills *down*; the chevron-left in the header walks one level let searchInput: TextInput | undefined = $state() let pickerRoot: HTMLElement | undefined = $state() - const instanceId = crypto.randomUUID() + const instanceId = generateRandomString(8) const listboxId = `pkr-list-${instanceId}` const idFor = (key: string) => `pkr-${instanceId}-${key.replace(/[^a-zA-Z0-9-]/g, '_')}` From 24e3ef27be8498fb820c228a52febf6a0a91b487 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 2 Jun 2026 09:44:04 +0200 Subject: [PATCH 10/61] fix(cli): stop git-sync promotion deploys from dropping triggers/schedules (#9403) * fix(cli): stop git-sync promotion deploys from dropping triggers/schedules Co-Authored-By: Claude Opus 4.8 * chore: bump git-sync hub script to hub/28261 (windmill-cli 1.713.2) Points LATEST_GIT_SYNC_SCRIPT_PATH at the republished sync-script-to-git-repo that pins windmill-cli@1.713.2, which carries the promotion include-derivation fix in this PR. Co-Authored-By: Claude Opus 4.8 --------- Co-authored-by: Claude Opus 4.8 --- backend/windmill-common/src/workspaces.rs | 2 +- cli/src/commands/sync/sync.ts | 13 +- cli/src/utils/git.ts | 56 +++- cli/test/git_unit.test.ts | 58 +++- cli/test/gitsync_promotion.test.ts | 375 ++++++++++++++++++++++ 5 files changed, 470 insertions(+), 34 deletions(-) diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index 5d6d2ed1b1..e20a896103 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -157,7 +157,7 @@ pub enum ObjectType { WorkspaceDependencies, } -pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28238/sync-script-to-git-repo-windmill"; +pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28261/sync-script-to-git-repo-windmill"; /// Prefix used to identify fork workspaces. A workspace whose id starts with this string is a /// fork of another workspace. diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index dda0f41035..2070ceb590 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -3053,12 +3053,13 @@ export async function gitDeploy( ...(opts.extraIncludes ?? []), ...includes.extraIncludes, ], - includeSchedules: opts.includeSchedules || includes.includeSchedules, - includeGroups: opts.includeGroups || includes.includeGroups, - includeUsers: opts.includeUsers || includes.includeUsers, - includeTriggers: opts.includeTriggers || includes.includeTriggers, - includeSettings: opts.includeSettings || includes.includeSettings, - includeKey: opts.includeKey || includes.includeKey, + // Workspace-wide mode force-includes the deployed default-excluded kinds + // (full mirror). Individual-branch/promotion mode forces nothing — these + // keys stay ABSENT so pull resolves them from the promotion target's + // effective wmill.yaml filters. Spreading (not setting `false`) is what + // makes the deferral work: an explicit `false` would clobber the effective + // config in pull's Object.assign-based option merge. + ...includes.forcedIncludes, promotion, } as any); } diff --git a/cli/src/utils/git.ts b/cli/src/utils/git.ts index 6e0ba55ec9..0d2e8cb8a8 100644 --- a/cli/src/utils/git.ts +++ b/cli/src/utils/git.ts @@ -252,21 +252,44 @@ export function gitSyncIncludePattern( } } -export interface GitSyncDeployIncludes { - extraIncludes: string[]; +// `forcedIncludes` carries ONLY the include-* flags that must be force-set to +// true (overriding the repo's wmill.yaml). Kinds not present are intentionally +// omitted (never set to false) so the caller can spread this object and let +// the repo's effective config govern the rest — see deriveGitSyncDeployIncludes. +export type GitSyncForcedIncludes = Partial<{ includeSchedules: boolean; includeGroups: boolean; includeUsers: boolean; includeTriggers: boolean; includeSettings: boolean; includeKey: boolean; +}>; + +export interface GitSyncDeployIncludes { + extraIncludes: string[]; + forcedIncludes: GitSyncForcedIncludes; } // Mirrors the hub script's wmill_sync_pull include-derivation: build the -// --extra-includes set from the deployed items, and (only in workspace-wide -// mode — never with --use-individual-branch) opt object kinds that are -// excluded by default back in. Replaces the script's regexFromPath + +// --extra-includes set from the deployed items, and decide which default- +// excluded object kinds (triggers, schedules, groups, users, settings, key) +// must be force-included in the pull. Replaces the script's regexFromPath + // per-kind --include-* construction so the hub script can drop both. +// +// Branch-mode distinction (this is load-bearing — see the trigger-promotion +// bug it fixes): +// - Workspace-wide mode: the repo is a full mirror of the workspace, so a +// deployed object of a default-excluded kind MUST be re-included, even if +// wmill.yaml would otherwise skip it. We force the flag on. +// - Individual-branch (promotion) mode: the repo is a filtered prod surface +// whose own wmill.yaml filters decide what gets promoted. We force NOTHING +// here and the keys stay absent, so the caller's pull resolves them from +// the target's effective config (a deployed trigger lands iff the target +// includes triggers). Forcing `false` (the original behavior) did NOT +// defer — it CLOBBERED the effective config via Object.assign in pull's +// option merge, silently dropping kinds the target actually wanted (e.g. a +// deployed trigger when the target has includeTriggers: true), and the +// server then omitted the object from the tarball entirely. export function deriveGitSyncDeployIncludes( items: GitSyncDeployItem[], useIndividualBranch: boolean, @@ -283,18 +306,19 @@ export function deriveGitSyncDeployIncludes( } } - const has = (pred: (t: string) => boolean) => - !useIndividualBranch && items.some((i) => pred(i.path_type)); + const forcedIncludes: GitSyncForcedIncludes = {}; + if (!useIndividualBranch) { + const has = (pred: (t: string) => boolean) => + items.some((i) => pred(i.path_type)); + if (has((t) => t === "schedule")) forcedIncludes.includeSchedules = true; + if (has((t) => t === "group")) forcedIncludes.includeGroups = true; + if (has((t) => t === "user")) forcedIncludes.includeUsers = true; + if (has((t) => t.includes("trigger"))) forcedIncludes.includeTriggers = true; + if (has((t) => t === "settings")) forcedIncludes.includeSettings = true; + if (has((t) => t === "key")) forcedIncludes.includeKey = true; + } - return { - extraIncludes, - includeSchedules: has((t) => t === "schedule"), - includeGroups: has((t) => t === "group"), - includeUsers: has((t) => t === "user"), - includeTriggers: has((t) => t.includes("trigger")), - includeSettings: has((t) => t === "settings"), - includeKey: has((t) => t === "key"), - }; + return { extraIncludes, forcedIncludes }; } function git( diff --git a/cli/test/git_unit.test.ts b/cli/test/git_unit.test.ts index f55e8d46b4..ebbc838f97 100644 --- a/cli/test/git_unit.test.ts +++ b/cli/test/git_unit.test.ts @@ -254,7 +254,7 @@ describe("deriveGitSyncDeployIncludes", () => { ]); }); - test("workspace-wide mode opts excluded kinds back in", () => { + test("workspace-wide mode force-includes deployed default-excluded kinds", () => { const r = deriveGitSyncDeployIncludes( [ { path_type: "schedule", path: "f/s" }, @@ -266,15 +266,36 @@ describe("deriveGitSyncDeployIncludes", () => { ], false ); - expect(r.includeSchedules).toBe(true); - expect(r.includeGroups).toBe(true); - expect(r.includeTriggers).toBe(true); - expect(r.includeSettings).toBe(true); - expect(r.includeKey).toBe(true); - expect(r.includeUsers).toBe(true); + // Full-mirror repo: a deployed object of a default-excluded kind must be + // re-included even if wmill.yaml would skip it, so the flag is forced on. + expect(r.forcedIncludes).toEqual({ + includeSchedules: true, + includeGroups: true, + includeTriggers: true, + includeSettings: true, + includeKey: true, + includeUsers: true, + }); }); - test("individual-branch mode NEVER sets include flags (matches hub script)", () => { + test("workspace-wide mode only forces the kinds actually deployed", () => { + const r = deriveGitSyncDeployIncludes( + [{ path_type: "script", path: "f/s" }], + false + ); + // Scripts are included by default — nothing to force. + expect(r.forcedIncludes).toEqual({}); + }); + + test("individual-branch (promotion) mode forces NOTHING — defers to wmill.yaml", () => { + // Regression: these flags used to be force-disabled (set to false) in + // promotion mode, which CLOBBERED the promotion target's effective + // wmill.yaml config (an explicit false wins in pull's Object.assign merge). + // The server then stripped the object from the tarball, the pull wrote + // nothing, and `git add '**'` failed with "pathspec did not match + // any files". Forcing nothing leaves the keys absent so the target's + // effective filters govern; extraIncludes still scopes the pull to the + // changed object. const r = deriveGitSyncDeployIncludes( [ { path_type: "schedule", path: "f/s" }, @@ -282,12 +303,27 @@ describe("deriveGitSyncDeployIncludes", () => { ], true ); - expect(r.includeSchedules).toBe(false); - expect(r.includeTriggers).toBe(false); - // extra-includes are still derived regardless of branch mode + expect(r.forcedIncludes).toEqual({}); expect(r.extraIncludes).toContain("f/s.schedule.*"); expect(r.extraIncludes).toContain("f/t.kafka_trigger.*"); }); + + test("regression: http_trigger promotion deploy does not clobber the target's includeTriggers", () => { + // Brad's scenario: an HTTP trigger is deployed and the promotion repo uses + // individual branches. path_type is "httptrigger" (the no-underscore value + // the backend puts on item.path_type — see git_sync_ee.rs + // insert_path_type_and_return_message). includeTriggers must NOT be forced + // false here, so the target's effective includeTriggers (true in Brad's + // config) is honored and the trigger file is pulled and committed. + const r = deriveGitSyncDeployIncludes( + [{ path_type: "httptrigger", path: "f/platform/on_call_chat_http_route" }], + true + ); + expect(r.forcedIncludes.includeTriggers).toBeUndefined(); + expect(r.extraIncludes).toContain( + "f/platform/on_call_chat_http_route.http_trigger.*" + ); + }); }); // ============================================================================= diff --git a/cli/test/gitsync_promotion.test.ts b/cli/test/gitsync_promotion.test.ts index 709ae11fc2..e33968d0fa 100644 --- a/cli/test/gitsync_promotion.test.ts +++ b/cli/test/gitsync_promotion.test.ts @@ -22,6 +22,18 @@ import { withTestBackend } from "./test_backend.ts"; import { shouldSkipOnCI } from "./cargo_backend.ts"; import { addWorkspace } from "../workspace.ts"; +// The HTTP-trigger promotion test creates an http_trigger, whose API routes are +// behind the `http_trigger` cargo feature — NOT in the default EE test feature +// set. The shared test backend reads TEST_FEATURES at construction (first +// `withTestBackend` call), so appending here at module load enables it. Guarded +// on shouldSkipOnCI() so we only widen the build when these EE tests actually +// run (i.e. EE_LICENSE_KEY present); minimal CI builds stay untouched. +if (!shouldSkipOnCI()) { + process.env["TEST_FEATURES"] = [process.env["TEST_FEATURES"], "http_trigger"] + .filter(Boolean) + .join(","); +} + function git(cwd: string, ...args: string[]): string { return execFileSync("git", args, { cwd, encoding: "utf8" }).trim(); } @@ -44,6 +56,24 @@ function remoteHead(bareDir: string, branch: string): string { ).trim(); } +// True if `filePath` exists in the tree of `branch` on the bare remote. +function fileExistsOnBranch( + bareDir: string, + branch: string, + filePath: string, +): boolean { + try { + execFileSync( + "git", + ["--git-dir", bareDir, "cat-file", "-e", `refs/heads/${branch}:${filePath}`], + { stdio: "ignore" }, + ); + return true; + } catch { + return false; + } +} + test.skipIf(shouldSkipOnCI())( "git-sync promotion: use_individual_branch pushes to wm_deploy branch, not main", async () => { @@ -201,6 +231,351 @@ test.skipIf(shouldSkipOnCI())( }, ); +/** + * Regression test for the promotion trigger-include bug (fix/gitsync-promotion- + * trigger-export): deploying a trigger (or any excluded-by-default kind: + * schedule, group, user, settings, key) with `use_individual_branch` must still + * land the object file on the `wm_deploy` branch. + * + * Root cause: `deriveGitSyncDeployIncludes` used to force the per-kind include + * flags (`includeTriggers` etc.) to false in individual-branch mode. The + * server-side tarball export STRIPS those object kinds entirely when their + * include flag is false (`if include_triggers { … }` in workspaces_export.rs), + * and `extraIncludes` is only a client-side filter over what the tarball + * already contains — it can't recover a file the server never sent. So the + * pull wrote no trigger file, the wm_deploy branch was created empty of the + * trigger, and production's `git add '**'` failed with "pathspec did not + * match any files". A script (always-included kind) never hit this — hence the + * dedicated trigger case here. + * + * Without the fix this test fails: the branch exists but the + * `*.http_trigger.yaml` file is absent from it. + */ +test.skipIf(shouldSkipOnCI())( + "git-sync promotion: use_individual_branch lands a trigger file on the wm_deploy branch", + async () => { + await withTestBackend(async (backend) => { + const ws = backend.workspace; // "test" + await addWorkspace( + { + remote: backend.baseUrl, + workspaceId: ws, + name: ws, + token: backend.token, + } as any, + { force: true, configDir: backend.testConfigDir }, + ); + + // --- 1. Bare "remote" seeded with an initial `main` commit --- + const bareDir = await mkdtemp(join(tmpdir(), "wmill_promo_trig_bare_")); + execFileSync("git", ["init", "--bare", "--initial-branch=main", bareDir]); + const seedDir = await mkdtemp(join(tmpdir(), "wmill_promo_trig_seed_")); + git(seedDir, "init", "--initial-branch=main"); + git(seedDir, "config", "user.email", "seed@windmill.dev"); + git(seedDir, "config", "user.name", "seed"); + await writeFile(join(seedDir, "README.md"), "# promo trigger test\n"); + git(seedDir, "add", "-A"); + git(seedDir, "commit", "-m", "seed"); + git(seedDir, "remote", "add", "origin", `file://${bareDir}`); + git(seedDir, "push", "-u", "origin", "main"); + const seedMain = remoteHead(bareDir, "main"); + + // --- 2. Workspace content: a script + an HTTP trigger pointing at it --- + await backend.apiRequest!(`/api/w/${ws}/folders/create`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ name: "promo", owners: [], extra_perms: {} }), + }); + await backend.apiRequest!(`/api/w/${ws}/scripts/create`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + path: "f/promo/foo", + summary: "", + description: "", + content: "export async function main() { return 1 }", + language: "bun", + }), + }); + const trigRes = await backend.apiRequest!(`/api/w/${ws}/http_triggers/create`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + path: "f/promo/hook", + script_path: "f/promo/foo", + route_path: "promo_hook", + is_flow: false, + http_method: "post", + authentication_method: "none", + is_static_website: false, + request_type: "sync", + }), + }); + // Guard against the route silently 404ing (the http_trigger cargo feature + // not being built) — otherwise the pull below would find nothing to sync + // and the real assertion would fail with a confusing message. + expect(trigRes.status).toBe(201); + + // --- 3. git_repository resource + git-sync config (triggers included) --- + await backend.apiRequest!(`/api/w/${ws}/resources/create`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + path: "u/test/promo_repo", + resource_type: "git_repository", + value: { url: `file://${bareDir}`, branch: "main", token: "" }, + }), + }); + await backend.updateGitSyncConfig!({ + git_sync_settings: { + repositories: [ + { + git_repo_resource_path: "u/test/promo_repo", + script_path: "f/**", + use_individual_branch: true, + group_by_folder: false, + settings: { + include_path: ["f/**"], + include_type: ["script", "trigger"], + }, + }, + ], + }, + }); + + // The backend sets path_type "httptrigger" (no underscore) on the deploy + // item — see DeployedObject::HttpTrigger => "httptrigger" in git_sync_ee.rs. + const deployItems = JSON.stringify([ + { + path_type: "httptrigger", + path: "f/promo/hook", + commit_msg: "deploy hook", + }, + ]); + + const work = await mkdtemp(join(tmpdir(), "wmill_promo_trig_work_")); + git(work, "clone", `file://${bareDir}`, "."); + // Option B semantic: in promotion mode the deploy forces NOTHING — the + // trigger lands only because THIS target's effective wmill.yaml opts + // triggers in. (Reverting the source fix re-introduces the force-`false` + // that clobbers this `includeTriggers: true`, so the file is dropped.) + await writeFile( + join(work, "wmill.yaml"), + "defaultTs: bun\nincludes:\n - f/**\nexcludes: []\nincludeTriggers: true\n", + ); + const res = await backend.runCLICommand( + [ + "sync", + "git-deploy", + "--repository", + "u/test/promo_repo", + "--use-individual-branch", + "--git-deploy-items", + deployItems, + ], + work, + ); + expect(res.code).toBe(0); + + // Caller-half (mirrors the hub script): stage what the pull wrote, commit + // on the checked-out wm_deploy branch, push. + git(work, "config", "user.email", "test@windmill.dev"); + git(work, "config", "user.name", "test"); + git(work, "add", "-A"); + try { + git(work, "diff", "--cached", "--quiet"); + } catch { + git(work, "commit", "-m", "deploy hook"); + } + git(work, "push", "--porcelain", "-u", "origin", "HEAD"); + + const expectedBranch = `refs/heads/wm_deploy/${ws}/httptrigger/f__promo__hook`; + expect(remoteBranches(bareDir)).toContain(expectedBranch); + // The regression: the trigger file MUST be present on the branch. Without + // the fix the include flag is false, the server strips the trigger from + // the tarball, the pull writes nothing, and this file is absent. + expect( + fileExistsOnBranch( + bareDir, + `wm_deploy/${ws}/httptrigger/f__promo__hook`, + "f/promo/hook.http_trigger.yaml", + ), + ).toBe(true); + // Base branch untouched (individual-branch never pushes to the base). + expect(remoteHead(bareDir, "main")).toBe(seedMain); + + await rm(bareDir, { recursive: true, force: true }); + await rm(seedDir, { recursive: true, force: true }); + await rm(work, { recursive: true, force: true }); + }); + }, +); + +/** + * Same regression as the HTTP-trigger case above, for a `schedule` — a + * different excluded-by-default kind that exercises a DISTINCT path: its own + * include flag (`includeSchedules`), its own server-side `if include_schedules` + * tarball-strip branch, and its own `.schedule.yaml` extension. Unlike triggers + * it needs no extra cargo feature, so it guards the fix even where the + * trigger-specific features aren't built. + * + * Without the fix this test fails: the branch exists but the + * `*.schedule.yaml` file is absent from it. + */ +test.skipIf(shouldSkipOnCI())( + "git-sync promotion: use_individual_branch lands a schedule file on the wm_deploy branch", + async () => { + await withTestBackend(async (backend) => { + const ws = backend.workspace; // "test" + await addWorkspace( + { + remote: backend.baseUrl, + workspaceId: ws, + name: ws, + token: backend.token, + } as any, + { force: true, configDir: backend.testConfigDir }, + ); + + // --- 1. Bare "remote" seeded with an initial `main` commit --- + const bareDir = await mkdtemp(join(tmpdir(), "wmill_promo_sched_bare_")); + execFileSync("git", ["init", "--bare", "--initial-branch=main", bareDir]); + const seedDir = await mkdtemp(join(tmpdir(), "wmill_promo_sched_seed_")); + git(seedDir, "init", "--initial-branch=main"); + git(seedDir, "config", "user.email", "seed@windmill.dev"); + git(seedDir, "config", "user.name", "seed"); + await writeFile(join(seedDir, "README.md"), "# promo schedule test\n"); + git(seedDir, "add", "-A"); + git(seedDir, "commit", "-m", "seed"); + git(seedDir, "remote", "add", "origin", `file://${bareDir}`); + git(seedDir, "push", "-u", "origin", "main"); + const seedMain = remoteHead(bareDir, "main"); + + // --- 2. Workspace content: a script + a (disabled) schedule for it --- + await backend.apiRequest!(`/api/w/${ws}/folders/create`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ name: "promo", owners: [], extra_perms: {} }), + }); + await backend.apiRequest!(`/api/w/${ws}/scripts/create`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + path: "f/promo/foo", + summary: "", + description: "", + content: "export async function main() { return 1 }", + language: "bun", + }), + }); + const schedRes = await backend.apiRequest!(`/api/w/${ws}/schedules/create`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + path: "f/promo/sched", + schedule: "0 0 12 * * *", + timezone: "UTC", + script_path: "f/promo/foo", + is_flow: false, + args: {}, + enabled: false, + }), + }); + expect(schedRes.status).toBe(200); + + // --- 3. git_repository resource + git-sync config (schedules included) --- + await backend.apiRequest!(`/api/w/${ws}/resources/create`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + path: "u/test/promo_repo", + resource_type: "git_repository", + value: { url: `file://${bareDir}`, branch: "main", token: "" }, + }), + }); + await backend.updateGitSyncConfig!({ + git_sync_settings: { + repositories: [ + { + git_repo_resource_path: "u/test/promo_repo", + script_path: "f/**", + use_individual_branch: true, + group_by_folder: false, + settings: { + include_path: ["f/**"], + include_type: ["script", "schedule"], + }, + }, + ], + }, + }); + + const deployItems = JSON.stringify([ + { + path_type: "schedule", + path: "f/promo/sched", + commit_msg: "deploy sched", + }, + ]); + + const work = await mkdtemp(join(tmpdir(), "wmill_promo_sched_work_")); + git(work, "clone", `file://${bareDir}`, "."); + // Option B semantic: in promotion mode the deploy forces NOTHING — the + // schedule lands only because THIS target's effective wmill.yaml opts + // schedules in. (Reverting the source fix re-introduces the force-`false` + // that clobbers this `includeSchedules: true`, so the file is dropped.) + await writeFile( + join(work, "wmill.yaml"), + "defaultTs: bun\nincludes:\n - f/**\nexcludes: []\nincludeSchedules: true\n", + ); + const res = await backend.runCLICommand( + [ + "sync", + "git-deploy", + "--repository", + "u/test/promo_repo", + "--use-individual-branch", + "--git-deploy-items", + deployItems, + ], + work, + ); + expect(res.code).toBe(0); + + // Caller-half (mirrors the hub script): stage what the pull wrote, commit + // on the checked-out wm_deploy branch, push. + git(work, "config", "user.email", "test@windmill.dev"); + git(work, "config", "user.name", "test"); + git(work, "add", "-A"); + try { + git(work, "diff", "--cached", "--quiet"); + } catch { + git(work, "commit", "-m", "deploy sched"); + } + git(work, "push", "--porcelain", "-u", "origin", "HEAD"); + + const expectedBranch = `refs/heads/wm_deploy/${ws}/schedule/f__promo__sched`; + expect(remoteBranches(bareDir)).toContain(expectedBranch); + // The regression: the schedule file MUST be present on the branch. Without + // the fix the include flag is false, the server strips the schedule from + // the tarball, the pull writes nothing, and this file is absent. + expect( + fileExistsOnBranch( + bareDir, + `wm_deploy/${ws}/schedule/f__promo__sched`, + "f/promo/sched.schedule.yaml", + ), + ).toBe(true); + // Base branch untouched (individual-branch never pushes to the base). + expect(remoteHead(bareDir, "main")).toBe(seedMain); + + await rm(bareDir, { recursive: true, force: true }); + await rm(seedDir, { recursive: true, force: true }); + await rm(work, { recursive: true, force: true }); + }); + }, +); + /** * Regression test for WIN-1997: forking a workspace with git sync configured * must publish a `wm-fork//` branch to the remote. From 8ad699d27b6ee9d28fadc41dfaecd2609788d5c5 Mon Sep 17 00:00:00 2001 From: Aldrin Jenson Date: Tue, 2 Jun 2026 04:16:11 -0400 Subject: [PATCH 11/61] Refresh slim image runtime packages (#9396) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Refresh slim runtime packages * fix(docker): bump pre-baked python to 3.12.12, drop redundant pip/setuptools upgrade Align the slim images' pre-baked uv-managed Python with the backend default (PyVAlias::Py312), which previously requested 3.12 while the image baked 3.11.10 — a minor mismatch that made the pre-bake unusable (every default job re-downloaded 3.12 at runtime). Pinning 3.12.12 (latest 3.12 in uv's list) also drops bundled setuptools entirely and ships current pip via python-build-standalone, so the explicit `uv pip install --upgrade pip setuptools` step is now redundant and removed. Also remove the dead PYTHON_IMAGE ARG from RHEL8/RHEL9 Dockerfiles (declared but never referenced in any FROM stage). Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Ruben Fiszel Co-authored-by: Claude Opus 4.8 (1M context) --- docker/DockerfileSlim | 4 ++-- docker/DockerfileSlimEe | 4 ++-- docker/RHEL8/Dockerfile | 1 - docker/RHEL9/Dockerfile | 1 - 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/docker/DockerfileSlim b/docker/DockerfileSlim index cd8ab1a33f..3301d73506 100644 --- a/docker/DockerfileSlim +++ b/docker/DockerfileSlim @@ -26,7 +26,7 @@ RUN make FROM ${DEBIAN_IMAGE} ARG APP=/usr/src/app -ARG LATEST_STABLE_PY=3.11.10 +ARG LATEST_STABLE_PY=3.12.12 # UV configuration ENV UV_CACHE_DIR=/tmp/windmill/cache/uv @@ -39,7 +39,7 @@ ENV PATH=/usr/local/bin:/root/.local/bin:/tmp/.local/bin:$PATH # Install system dependencies RUN apt-get update \ - && apt-get install -y --no-install-recommends ca-certificates wget curl git jq unzip unixodbc xmlsec1 gnupg lsb-release libgnutls30 \ + && apt-get install -y --no-install-recommends ca-certificates wget curl git jq unzip unixodbc xmlsec1 gnupg lsb-release libgnutls30 libgcrypt20 \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* diff --git a/docker/DockerfileSlimEe b/docker/DockerfileSlimEe index fb807d38a9..48cd3698a1 100644 --- a/docker/DockerfileSlimEe +++ b/docker/DockerfileSlimEe @@ -26,7 +26,7 @@ RUN make FROM ${DEBIAN_IMAGE} ARG APP=/usr/src/app -ARG LATEST_STABLE_PY=3.11.10 +ARG LATEST_STABLE_PY=3.12.12 # UV configuration ENV UV_CACHE_DIR=/tmp/windmill/cache/uv @@ -39,7 +39,7 @@ ENV PATH=/usr/local/bin:/root/.local/bin:/tmp/.local/bin:$PATH # Install system dependencies RUN apt-get update \ - && apt-get install -y --no-install-recommends ca-certificates wget curl git jq unzip unixodbc xmlsec1 gnupg lsb-release libgnutls30 \ + && apt-get install -y --no-install-recommends ca-certificates wget curl git jq unzip unixodbc xmlsec1 gnupg lsb-release libgnutls30 libgcrypt20 \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* diff --git a/docker/RHEL8/Dockerfile b/docker/RHEL8/Dockerfile index 500050de67..57b74b75af 100644 --- a/docker/RHEL8/Dockerfile +++ b/docker/RHEL8/Dockerfile @@ -1,6 +1,5 @@ ARG DEBIAN_IMAGE=debian:bookworm-slim ARG RUST_IMAGE=registry.access.redhat.com/ubi8/ubi:latest -ARG PYTHON_IMAGE=python:3.11.10-slim-bookworm FROM ${RUST_IMAGE} AS rust_base diff --git a/docker/RHEL9/Dockerfile b/docker/RHEL9/Dockerfile index a0fff8dd91..21816ba8a5 100644 --- a/docker/RHEL9/Dockerfile +++ b/docker/RHEL9/Dockerfile @@ -1,6 +1,5 @@ ARG DEBIAN_IMAGE=debian:bookworm-slim ARG RUST_IMAGE=registry.access.redhat.com/ubi9/ubi:latest -ARG PYTHON_IMAGE=python:3.11.10-slim-bookworm FROM ${RUST_IMAGE} AS rust_base From 2ac198396eced8bad44037fcce8b9cca987b2e4b Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 2 Jun 2026 10:22:19 +0200 Subject: [PATCH 12/61] chore(main): release 1.714.0 (#9390) * chore(main): release 1.714.0 * Apply automatic changes --------- Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com> --- CHANGELOG.md | 26 +++ backend/Cargo.lock | 186 +++++++++--------- backend/Cargo.toml | 4 +- .../parsers/windmill-parser-wasm/Cargo.lock | 48 ++--- .../parsers/windmill-parser-wasm/Cargo.toml | 2 +- backend/windmill-api/openapi.yaml | 2 +- benchmarks/lib.ts | 2 +- cli/src/main.ts | 2 +- frontend/package-lock.json | 4 +- frontend/package.json | 2 +- lsp/Pipfile | 2 +- openflow.openapi.yaml | 2 +- .../WindmillClient/WindmillClient.psd1 | 2 +- python-client/wmill/pyproject.toml | 2 +- typescript-client/jsr.json | 2 +- typescript-client/package.json | 2 +- version.txt | 2 +- 17 files changed, 159 insertions(+), 133 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c725b0f69a..193472c590 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,31 @@ # Changelog +## [1.714.0](https://github.com/windmill-labs/windmill/compare/v1.713.1...v1.714.0) (2026-06-02) + + +### Features + +* add global ai chat test tools ([#9391](https://github.com/windmill-labs/windmill/issues/9391)) ([5c20d6b](https://github.com/windmill-labs/windmill/commit/5c20d6b4f79f2ccc1987ce7fdaf74e6b8f697846)) +* add workspace datatable tools to global AI chat mode ([#9395](https://github.com/windmill-labs/windmill/issues/9395)) ([943ef6e](https://github.com/windmill-labs/windmill/commit/943ef6eb2089f4b744cfa7945ce47f7f3b361ec7)) +* **flow-ai:** constrain flow-group colors to the NoteColor palette ([#9343](https://github.com/windmill-labs/windmill/issues/9343)) ([e4213c1](https://github.com/windmill-labs/windmill/commit/e4213c1ab8c448f492f372580f5c9df37e33fffc)) +* **frontend:** surface local drafts in drawer editors with an unsaved-changes banner ([#9335](https://github.com/windmill-labs/windmill/issues/9335)) ([075faab](https://github.com/windmill-labs/windmill/commit/075faabf3bba16a10a02ae3973008e5a13473085)) +* handle CTRL_BREAK_EVENT for graceful shutdown on Windows ([#9400](https://github.com/windmill-labs/windmill/issues/9400)) ([2e14456](https://github.com/windmill-labs/windmill/commit/2e1445616a412c5112ad2247b4087c7ddc218845)) +* refine ask-user-question chat display and keyboard nav ([#9392](https://github.com/windmill-labs/windmill/issues/9392)) ([1275487](https://github.com/windmill-labs/windmill/commit/1275487f028d4c74a9eeb18981ed05c225505be0)) +* sessions page with isolated AI chat + flow editor ([#9034](https://github.com/windmill-labs/windmill/issues/9034)) ([eadeac2](https://github.com/windmill-labs/windmill/commit/eadeac248bd022c2796cfe638eb617c6143b8fc4)) + + +### Bug Fixes + +* **cli:** make encryption key push non-interactive-safe + add --skip-reencrypt-on-key-change ([#9402](https://github.com/windmill-labs/windmill/issues/9402)) ([e356bb1](https://github.com/windmill-labs/windmill/commit/e356bb1f5df92eca3fbb0ca2114b9f4c32d4c496)) +* **cli:** stop git-sync promotion deploys from dropping triggers/schedules ([#9403](https://github.com/windmill-labs/windmill/issues/9403)) ([24e3ef2](https://github.com/windmill-labs/windmill/commit/24e3ef27be8498fb820c228a52febf6a0a91b487)) +* **frontend:** align Monaco editor font size with text-xs ([#9161](https://github.com/windmill-labs/windmill/issues/9161)) ([de76668](https://github.com/windmill-labs/windmill/commit/de76668c10c04abe8771a8ca7bba7b2259819a1c)) +* resolve username rename failing on apps with runnable deps ([#9401](https://github.com/windmill-labs/windmill/issues/9401)) ([e8ad53d](https://github.com/windmill-labs/windmill/commit/e8ad53dae92597f5a1a8b76f38a7d8c24f578a47)) + + +### Performance Improvements + +* **python:** add --compile-bytecode to uv pip install ([#9393](https://github.com/windmill-labs/windmill/issues/9393)) ([c19441b](https://github.com/windmill-labs/windmill/commit/c19441bc8cb2da064e4ad44d77dc04ab8bbb22ec)) + ## [1.713.1](https://github.com/windmill-labs/windmill/compare/v1.713.0...v1.713.1) (2026-06-01) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 079944ac88..e39f97cd1f 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -1148,7 +1148,7 @@ dependencies = [ "pin-project-lite", "rustls 0.21.12", "rustls 0.23.35", - "rustls-native-certs 0.8.3", + "rustls-native-certs 0.8.4", "rustls-pki-types", "tokio", "tokio-rustls 0.26.4", @@ -4984,9 +4984,9 @@ dependencies = [ [[package]] name = "generator" -version = "0.8.8" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52f04ae4152da20c76fe800fa48659201d5cf627c5149ca0b707b69d7eef6cf9" +checksum = "b3b854b0e584ead1a33f18b2fcad7cf7be18b3875c78816b753639aa501513ae" dependencies = [ "cc", "cfg-if", @@ -5797,7 +5797,7 @@ dependencies = [ "hyper-util", "log", "rustls 0.23.35", - "rustls-native-certs 0.8.3", + "rustls-native-certs 0.8.4", "tokio", "tokio-rustls 0.26.4", "tower-service", @@ -6639,7 +6639,7 @@ dependencies = [ "bitflags 2.11.1", "libc", "plain", - "redox_syscall 0.8.0", + "redox_syscall 0.8.1", ] [[package]] @@ -9296,9 +9296,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c7591fa2c6b601dfcfe5f043f65a1c39fcdf50efefcd7f1572e538c1f4b398d" +checksum = "5b44b894f2a6e36457d665d1e08c3866add6ed5e70050c1b4ba8a8ddedb02ce7" dependencies = [ "bitflags 2.11.1", ] @@ -9432,7 +9432,7 @@ dependencies = [ "pin-project-lite", "quinn", "rustls 0.23.35", - "rustls-native-certs 0.8.3", + "rustls-native-certs 0.8.4", "rustls-pki-types", "serde", "serde_json", @@ -9956,9 +9956,9 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" dependencies = [ "openssl-probe 0.2.1", "rustls-pki-types", @@ -10006,7 +10006,7 @@ dependencies = [ "log", "once_cell", "rustls 0.23.35", - "rustls-native-certs 0.8.3", + "rustls-native-certs 0.8.4", "rustls-platform-verifier-android", "rustls-webpki 0.103.13", "security-framework 3.7.0", @@ -12504,7 +12504,7 @@ dependencies = [ "httparse", "rand 0.8.5", "ring 0.17.14", - "rustls-native-certs 0.8.3", + "rustls-native-certs 0.8.4", "rustls-pki-types", "tokio", "tokio-rustls 0.26.4", @@ -12629,7 +12629,7 @@ dependencies = [ "percent-encoding", "pin-project", "prost", - "rustls-native-certs 0.8.3", + "rustls-native-certs 0.8.4", "socket2 0.5.10", "tokio", "tokio-rustls 0.26.4", @@ -13159,9 +13159,9 @@ checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" [[package]] name = "unicode-segmentation" -version = "1.13.2" +version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" [[package]] name = "unicode-width" @@ -13764,7 +13764,7 @@ dependencies = [ [[package]] name = "windmill" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "async-nats", @@ -13845,7 +13845,7 @@ dependencies = [ [[package]] name = "windmill-ai" -version = "1.713.1" +version = "1.714.0" dependencies = [ "async-stream", "async-trait", @@ -13878,7 +13878,7 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.713.1" +version = "1.714.0" dependencies = [ "axum 0.8.9", "chrono", @@ -13891,7 +13891,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "argon2", @@ -14029,7 +14029,7 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.713.1" +version = "1.714.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14052,7 +14052,7 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.713.1" +version = "1.714.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14065,7 +14065,7 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14091,7 +14091,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.713.1" +version = "1.714.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -14101,7 +14101,7 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.713.1" +version = "1.714.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14118,7 +14118,7 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.713.1" +version = "1.714.0" dependencies = [ "axum 0.8.9", "base64 0.22.1", @@ -14140,7 +14140,7 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14163,7 +14163,7 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.713.1" +version = "1.714.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14179,7 +14179,7 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.713.1" +version = "1.714.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14200,7 +14200,7 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.713.1" +version = "1.714.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14221,7 +14221,7 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.713.1" +version = "1.714.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14235,7 +14235,7 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "async-nats", @@ -14267,7 +14267,7 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14292,7 +14292,7 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.713.1" +version = "1.714.0" dependencies = [ "axum 0.8.9", "flate2", @@ -14310,7 +14310,7 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14332,7 +14332,7 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.713.1" +version = "1.714.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14352,7 +14352,7 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.713.1" +version = "1.714.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14382,7 +14382,7 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14410,7 +14410,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.713.1" +version = "1.714.0" dependencies = [ "lazy_static", "serde", @@ -14422,7 +14422,7 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.713.1" +version = "1.714.0" dependencies = [ "argon2", "axum 0.8.9", @@ -14447,7 +14447,7 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.713.1" +version = "1.714.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14461,7 +14461,7 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.713.1" +version = "1.714.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14494,7 +14494,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.713.1" +version = "1.714.0" dependencies = [ "chrono", "lazy_static", @@ -14508,7 +14508,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14527,7 +14527,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.713.1" +version = "1.714.0" dependencies = [ "aes-gcm", "aho-corasick", @@ -14628,7 +14628,7 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.713.1" +version = "1.714.0" dependencies = [ "chrono", "itertools 0.14.0", @@ -14647,7 +14647,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.713.1" +version = "1.714.0" dependencies = [ "regex", "serde", @@ -14662,7 +14662,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "astral-tokio-tar", @@ -14686,7 +14686,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "futures", @@ -14703,7 +14703,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.713.1" +version = "1.714.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -14719,7 +14719,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "async-trait", @@ -14740,7 +14740,7 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "async-trait", @@ -14771,7 +14771,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "arc-swap", @@ -14796,7 +14796,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "async-stream", @@ -14830,7 +14830,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "futures", @@ -14848,7 +14848,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.713.1" +version = "1.714.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -14857,7 +14857,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "lazy_static", @@ -14869,7 +14869,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "serde_json", @@ -14881,7 +14881,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "gosyn", @@ -14893,7 +14893,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "lazy_static", @@ -14905,7 +14905,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "serde_json", @@ -14917,7 +14917,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "nu-parser", @@ -14928,7 +14928,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -14939,7 +14939,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -14951,7 +14951,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "rustpython-ast", @@ -14962,7 +14962,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "async-recursion", @@ -14984,7 +14984,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "serde_json", @@ -14996,7 +14996,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "lazy_static", @@ -15010,7 +15010,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -15027,7 +15027,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "lazy_static", @@ -15040,7 +15040,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "serde", @@ -15052,7 +15052,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "lazy_static", @@ -15070,7 +15070,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -15086,7 +15086,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "rustpython-ast", @@ -15102,7 +15102,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "serde", @@ -15113,7 +15113,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "async-recursion", @@ -15151,7 +15151,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "const_format", @@ -15189,7 +15189,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.713.1" +version = "1.714.0" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -15200,7 +15200,7 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "async-recursion", @@ -15230,7 +15230,7 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "async-trait", @@ -15254,7 +15254,7 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "async-trait", @@ -15287,7 +15287,7 @@ dependencies = [ [[package]] name = "windmill-trigger-azure" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "async-trait", @@ -15320,7 +15320,7 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "async-trait", @@ -15340,7 +15340,7 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "async-trait", @@ -15374,7 +15374,7 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "async-trait", @@ -15410,7 +15410,7 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "async-trait", @@ -15433,7 +15433,7 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "async-trait", @@ -15457,7 +15457,7 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "async-nats", @@ -15481,7 +15481,7 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "async-trait", @@ -15516,7 +15516,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "async-trait", @@ -15544,7 +15544,7 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "async-trait", @@ -15569,7 +15569,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "bitflags 2.11.1", @@ -15588,7 +15588,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "async-once-cell", @@ -15698,7 +15698,7 @@ dependencies = [ [[package]] name = "windmill-worker-volumes" -version = "1.713.1" +version = "1.714.0" dependencies = [ "bytes", "futures", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 9c15e1341b..32ae3ce1fc 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.713.1" +version = "1.714.0" authors.workspace = true edition.workspace = true @@ -87,7 +87,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"] [workspace.package] -version = "1.713.1" +version = "1.714.0" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/parsers/windmill-parser-wasm/Cargo.lock b/backend/parsers/windmill-parser-wasm/Cargo.lock index 94b979af53..aa0b618798 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.lock +++ b/backend/parsers/windmill-parser-wasm/Cargo.lock @@ -6183,7 +6183,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill-common" -version = "1.713.1" +version = "1.714.0" dependencies = [ "aho-corasick", "anyhow", @@ -6263,7 +6263,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.713.1" +version = "1.714.0" dependencies = [ "proc-macro2", "quote", @@ -6275,7 +6275,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.713.1" +version = "1.714.0" dependencies = [ "convert_case", "serde", @@ -6284,7 +6284,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "lazy_static", @@ -6296,7 +6296,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "serde_json", @@ -6308,7 +6308,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "gosyn", @@ -6320,7 +6320,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "lazy_static", @@ -6332,7 +6332,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "serde_json", @@ -6344,7 +6344,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "nu-parser", @@ -6355,7 +6355,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6366,7 +6366,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6378,7 +6378,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6389,7 +6389,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "async-recursion", @@ -6411,7 +6411,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "serde_json", @@ -6423,7 +6423,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "lazy_static", @@ -6437,7 +6437,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "convert_case", @@ -6454,7 +6454,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "lazy_static", @@ -6467,7 +6467,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "serde", @@ -6479,7 +6479,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "lazy_static", @@ -6497,7 +6497,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -6513,7 +6513,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6529,7 +6529,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "getrandom 0.2.17", @@ -6561,7 +6561,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "serde", @@ -6572,7 +6572,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.713.1" +version = "1.714.0" dependencies = [ "anyhow", "bitflags", diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index e2bb2affad..ec9a764a03 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -12,7 +12,7 @@ resolver = "2" members = ["."] [workspace.package] -version = "1.713.1" +version = "1.714.0" edition = "2021" authors = ["Ruben Fiszel "] diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index f56944ab6b..ae05dcb985 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.713.1 + version: 1.714.0 title: Windmill API contact: diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index d3479c294e..dff2f0fa12 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.713.1"; +export const VERSION = "v1.714.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/src/main.ts b/cli/src/main.ts index 6e98f4d7cb..da4a46da71 100755 --- a/cli/src/main.ts +++ b/cli/src/main.ts @@ -89,7 +89,7 @@ export { token, }; -export const VERSION = "1.713.1"; +export const VERSION = "1.714.0"; // Re-exported from constants.ts to maintain backwards compatibility export { WM_FORK_PREFIX } from "./core/constants.ts"; diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 11156f4319..f4780b5871 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "@windmill-labs/components", - "version": "1.713.1", + "version": "1.714.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@windmill-labs/components", - "version": "1.713.1", + "version": "1.714.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { diff --git a/frontend/package.json b/frontend/package.json index 38340bebb9..a96e7e1a3e 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "@windmill-labs/components", - "version": "1.713.1", + "version": "1.714.0", "scripts": { "dev": "vite dev", "dev:ui-builder": "mv static/ui_builder static/ui_builder.dev-disabled 2>/dev/null || true ; trap 'mv static/ui_builder.dev-disabled static/ui_builder 2>/dev/null || true' EXIT ; vite dev", diff --git a/lsp/Pipfile b/lsp/Pipfile index 4b216ab6fd..af8a1add3b 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,7 +4,7 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.713.1" +wmill = ">=1.714.0" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index fc74ed6a20..34944d78de 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: '3.0.3' info: - version: 1.713.1 + version: 1.714.0 title: OpenFlow Spec contact: name: Ruben Fiszel diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index e6af8d10ce..c7b31520cc 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. - ModuleVersion = '1.713.1' + ModuleVersion = '1.714.0' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index 70cdeb996d..d82c98e4cf 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.713.1" +version = "1.714.0" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index d267b2dc58..7e0945ac7c 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.713.1", + "version": "1.714.0", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./sqlUtils.ts", "./client.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index 6065a2dde3..6c3778ce59 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.713.1", + "version": "1.714.0", "author": "Ruben Fiszel", "license": "Apache 2.0", "sideEffects": false, diff --git a/version.txt b/version.txt index ee288bb10c..1a0a78ba16 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.713.1 +1.714.0 From 73edebc833a981488a8ea116f4f13c020a011a6f Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 2 Jun 2026 12:23:39 +0200 Subject: [PATCH 13/61] fix(backend): route //native TypeScript previews to native workers (WIN-2007) (#9407) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(backend): route //native TypeScript previews to native workers Previewing a TypeScript script carrying the `//native` annotation was pushed with `language = bun` (what the editor sends), so the job was tagged `bun` and routed to a regular bun worker. A native-mode worker neither matches the `bun` tag nor accepts a non-native `script_lang` (worker.rs rejects with "cannot execute non-native job with language 'bun'"), so previewing a `//native` script on a native-only worker setup failed — even though the deployed version of the same script runs fine as `bunnative` / tag `nativets`. `push` now reconciles the preview language with the `//native` annotation for `JobPayload::Code`, mirroring the deploy-time logic in `worker_lockfiles`: `bun` + `//native` is promoted to `bunnative` (tag `nativets`), and `bunnative` without `//native` is demoted back to `bun`. This makes a preview run exactly like the deployed script would, and covers every preview entry point (run_preview_script, inline preview, codebase preview) since they all go through `JobPayload::Code`. Adds regression tests asserting the queued job's `script_lang`/`tag` for all four (declared language × annotation) combinations. Fixes WIN-2007 Co-Authored-By: Claude Opus 4.8 (1M context) * chore(backend): add sqlx cache for preview_native_tag test query The regression test's `sqlx::query!` for `v2_job` (tag, script_lang) needs a cached entry so `SQLX_OFFLINE=true` CI compiles it. Adds exactly one new cache file; no existing (OSS or EE) caches removed. Co-Authored-By: Claude Opus 4.8 (1M context) * test(backend): trim preview native-tag tests to the essentials Keep the core regression (bun + //native → bunnative/nativets) and the guard that plain bun previews are unaffected. Drop the two bunnative- declared cases, which only re-verified the mirrored demote logic and weren't the reported issue. The shared query is unchanged, so the sqlx cache stays valid. Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Claude Opus 4.8 (1M context) --- ...6f0322a83c01cc465742f54a21f8fe5f4f037.json | 60 +++++++++ backend/tests/preview_native_tag.rs | 122 ++++++++++++++++++ backend/windmill-queue/src/jobs.rs | 17 ++- 3 files changed, 198 insertions(+), 1 deletion(-) create mode 100644 backend/.sqlx/query-cce5e3e639faed8e42574730cc66f0322a83c01cc465742f54a21f8fe5f4f037.json create mode 100644 backend/tests/preview_native_tag.rs diff --git a/backend/.sqlx/query-cce5e3e639faed8e42574730cc66f0322a83c01cc465742f54a21f8fe5f4f037.json b/backend/.sqlx/query-cce5e3e639faed8e42574730cc66f0322a83c01cc465742f54a21f8fe5f4f037.json new file mode 100644 index 0000000000..29f31c62eb --- /dev/null +++ b/backend/.sqlx/query-cce5e3e639faed8e42574730cc66f0322a83c01cc465742f54a21f8fe5f4f037.json @@ -0,0 +1,60 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT tag, script_lang AS \"script_lang: ScriptLang\" FROM v2_job WHERE id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "tag", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "script_lang: ScriptLang", + "type_info": { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java", + "duckdb", + "ruby", + "rlang" + ] + } + } + } + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + false, + true + ] + }, + "hash": "cce5e3e639faed8e42574730cc66f0322a83c01cc465742f54a21f8fe5f4f037" +} diff --git a/backend/tests/preview_native_tag.rs b/backend/tests/preview_native_tag.rs new file mode 100644 index 0000000000..29aefa588a --- /dev/null +++ b/backend/tests/preview_native_tag.rs @@ -0,0 +1,122 @@ +/* + * Regression tests for WIN-2007. + * + * Previewing a TypeScript script carrying the `//native` annotation used to be + * pushed with `language = bun` (what the editor sends), so the job was tagged + * `bun` and routed to a regular bun worker. A native-mode worker neither matches + * the `bun` tag nor accepts a non-native `script_lang`, so previewing a `//native` + * script on a native-only worker setup failed even though the *deployed* version + * of the same script runs fine (as `bunnative` / tag `nativets`). + * + * `push` now reconciles the preview language with the `//native` annotation, + * mirroring the deploy-time logic in `worker_lockfiles`. These tests assert the + * queued job ends up with the right `script_lang` and `tag` for every combination + * of declared language and annotation. No worker is spawned — we only inspect the + * row `push` writes. + */ + +use sqlx::{Pool, Postgres}; +use windmill_common::{ + jobs::{JobPayload, RawCode}, + scripts::ScriptLang, +}; +use windmill_queue::PushIsolationLevel; + +async fn push_preview_and_get_row( + db: &Pool, + content: &str, + language: ScriptLang, +) -> (String, Option) { + let hm_args = std::collections::HashMap::new(); + + let job = JobPayload::Code(RawCode { + hash: None, + content: content.to_string(), + path: None, + language, + lock: None, + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() + .into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + modules: None, + tag: None, + }); + + let tx = PushIsolationLevel::IsolatedRoot(db.clone()); + let (uuid, tx) = windmill_queue::push( + db, + tx, + "test-workspace", + job, + windmill_queue::PushArgs::from(&hm_args), + /* user */ "test-user", + /* email */ "test@windmill.dev", + /* permissioned_as */ "u/test-user".to_string(), + /* token_prefix */ None, + /* scheduled_for */ None, + /* schedule_path */ None, + /* parent_job */ None, + /* root_job */ None, + /* flow_innermost_root_job */ None, + /* job_id */ None, + /* is_flow_step */ false, + /* same_worker */ false, + None, + true, + None, + None, + None, + None, + None, + false, + None, + None, + None, + ) + .await + .expect("push must succeed"); + tx.commit().await.unwrap(); + + let row = sqlx::query!( + r#"SELECT tag, script_lang AS "script_lang: ScriptLang" FROM v2_job WHERE id = $1"#, + uuid + ) + .fetch_one(db) + .await + .unwrap(); + (row.tag, row.script_lang) +} + +const NATIVE_CONTENT: &str = r#"//native + +export function main(x: number) { + return x; +} +"#; + +const PLAIN_CONTENT: &str = r#"export function main(x: number) { + return x; +} +"#; + +/// The reported case: editor sends `bun`, content has `//native`. The preview +/// must be promoted to `bunnative` so it tags `nativets` and a native worker +/// (which rejects non-native `script_lang`) can run it. +#[sqlx::test(fixtures("base"))] +async fn test_bun_with_native_annotation_becomes_nativets(db: Pool) { + let (tag, lang) = push_preview_and_get_row(&db, NATIVE_CONTENT, ScriptLang::Bun).await; + assert_eq!(lang, Some(ScriptLang::Bunnative)); + assert_eq!(tag, "nativets"); +} + +/// Guard: a plain bun preview (no `//native`) must stay `bun` / tag `bun`, so +/// the promotion above doesn't broadly retag normal previews. +#[sqlx::test(fixtures("base"))] +async fn test_bun_without_native_annotation_stays_bun(db: Pool) { + let (tag, lang) = push_preview_and_get_row(&db, PLAIN_CONTENT, ScriptLang::Bun).await; + assert_eq!(lang, Some(ScriptLang::Bun)); + assert_eq!(tag, "bun"); +} diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 964fd558f1..7c4a832760 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -5058,7 +5058,7 @@ async fn push_inner<'c, 'd>( content, path, hash, - language, + mut language, lock, cache_ttl, cache_ignore_s3_path, @@ -5068,6 +5068,21 @@ async fn push_inner<'c, 'd>( debouncing_settings, modules, }) => { + // Reconcile the preview language with the `//native` annotation, mirroring the + // deploy-time logic in `worker_lockfiles`. The editor sends `bun` for a TypeScript + // script even when it carries `//native`, which would otherwise tag the preview as + // `bun` and route it to a regular bun worker. A native-mode worker neither matches + // the `bun` tag nor accepts a non-native `script_lang`, so previewing a `//native` + // script on a native-only worker setup fails. Normalizing to `bunnative` (tag + // `nativets`) makes the preview run exactly like the deployed script would. + if language == ScriptLang::Bun || language == ScriptLang::Bunnative { + let anns = windmill_common::worker::TypeScriptAnnotations::parse(&content); + if anns.native && language == ScriptLang::Bun { + language = ScriptLang::Bunnative; + } else if !anns.native && language == ScriptLang::Bunnative { + language = ScriptLang::Bun; + } + } // Inject modules into job args as _MODULES so the worker can extract them if let Some(ref modules) = modules { match serde_json::to_string(modules).and_then(|s| RawValue::from_string(s)) { From 9e6559a6f688cc8d982277b19920219ea6d0fd8e Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 2 Jun 2026 14:44:30 +0200 Subject: [PATCH 14/61] fix(nsjail): raise python download fd limit for --compile-bytecode (WIN-2009) (#9414) #9393 added `--compile-bytecode` to the uv pip install run inside the python download nsjail. uv spawns a Python interpreter that compiles .py files with parallelism scaling to the host CPU count, opening many file descriptors at once. The download nsjail capped `rlimit_nofile` at 64, which is exhausted on high-core machines, failing every install with "Failed to bytecode-compile ... Too many open files (os error 24)". Low-core VMs never hit the cap, so this surfaced only as a regression on larger workers after upgrading. Raise `rlimit_nofile` to 10000, matching the runtime configs (run.python3 / run.ansible) that already use that value. Fixes WIN-2009 Co-authored-by: Claude Opus 4.8 (1M context) --- backend/windmill-worker/nsjail/download.py.config.proto | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/backend/windmill-worker/nsjail/download.py.config.proto b/backend/windmill-worker/nsjail/download.py.config.proto index e56ef66de0..217fe5763a 100644 --- a/backend/windmill-worker/nsjail/download.py.config.proto +++ b/backend/windmill-worker/nsjail/download.py.config.proto @@ -8,7 +8,11 @@ time_limit: 900 rlimit_as: 2048 rlimit_cpu: 1000 rlimit_fsize: 1024 -rlimit_nofile: 64 +# uv's --compile-bytecode spawns a Python interpreter that compiles .py files +# with parallelism scaling to the host's CPU count, opening many fds at once. +# A low cap (was 64) is exhausted on high-core machines -> "Too many open files". +# Matches the runtime configs (run.python3/run.ansible) which already use 10000. +rlimit_nofile: 10000 envar: "HOME=/user" envar: "LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH" From ab2a15b2a859096eabde718bf6e60289ae187118 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 2 Jun 2026 14:47:45 +0200 Subject: [PATCH 15/61] fix(triggers): prevent Zoom challenge handler from being used as a signing oracle (#9413) The Zoom URL-validation challenge handler in `handle_challenge_request` would HMAC-sign any arbitrary `plainToken` and return the result. Since Zoom webhook verification checks `HMAC-SHA256(secret, "v0:{ts}:{body}")`, an attacker could craft a `plainToken` in that format to obtain a valid signature for a forged body, bypassing authentication on a later request. Unlike the Twitch handler, the Zoom handler verifies no signature on the challenge request (Zoom's protocol does not include one). Reject any `plainToken` containing `:` or longer than 128 chars: legitimate Zoom validation tokens are short random hex strings that never contain colons, while the exploit requires the colon-bearing `v0:{ts}:{body}` format. Fixes WIN-2008 Co-authored-by: Claude Opus 4.8 (1M context) --- .../src/http_trigger_auth.rs | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/backend/windmill-trigger-http/src/http_trigger_auth.rs b/backend/windmill-trigger-http/src/http_trigger_auth.rs index 19766cdbc2..10927d1ef9 100644 --- a/backend/windmill-trigger-http/src/http_trigger_auth.rs +++ b/backend/windmill-trigger-http/src/http_trigger_auth.rs @@ -337,6 +337,20 @@ mod zoom { return Ok(None); } + // Prevent this challenge endpoint from being used as a signing oracle. + // Legitimate Zoom validation tokens are short random hex strings that + // never contain colons. The exploit requires crafting a plainToken in the + // `v0:{timestamp}:{body}` webhook-signing format (always containing colons) + // to obtain a valid signature for an arbitrary body. Reject any token that + // does not look like a legitimate Zoom validation token. + if zoom_request_body.payload.plain_token.contains(':') + || zoom_request_body.payload.plain_token.len() > 128 + { + return Err(AuthenticationError::InvalidChallengeResponse( + "Zoom: invalid plainToken format".to_string(), + )); + } + let hmac_signature = calculate_hmac_signature( HmacAlgorithm::Sha256, &signature_config_data.secret_key, @@ -1540,6 +1554,52 @@ mod tests { assert!(response.is_none()); } + #[test] + fn test_zoom_challenge_normal_token_succeeds() { + // A legitimate Zoom validation token is a short random alphanumeric string. + let payload = r#"{"event":"endpoint.url_validation","event_ts":1234567890,"payload":{"plainToken":"qgg8vlvZRS6UYooatFL8Aw"}}"#; + + let handler = WebhookType::Zoom.get_webhook_handler().unwrap(); + let config_data = SignatureConfigData { secret_key: "zoom_secret" }; + let response = handler + .handle_challenge_request(&HeaderMap::new(), &config_data, payload) + .unwrap(); + assert!(response.is_some()); + } + + #[test] + fn test_zoom_challenge_token_with_colons_rejected() { + // Exploit attempt: a plainToken crafted in the `v0:{ts}:{body}` signing format + // would let an attacker obtain a valid webhook signature for an arbitrary body. + let payload = r#"{"event":"endpoint.url_validation","event_ts":1234567890,"payload":{"plainToken":"v0:1234567890:{\"forged\":\"body\"}"}}"#; + + let handler = WebhookType::Zoom.get_webhook_handler().unwrap(); + let config_data = SignatureConfigData { secret_key: "zoom_secret" }; + let result = handler.handle_challenge_request(&HeaderMap::new(), &config_data, payload); + assert!(matches!( + result, + Err(AuthenticationError::InvalidChallengeResponse(_)) + )); + } + + #[test] + fn test_zoom_challenge_token_too_long_rejected() { + // A plainToken exceeding 128 chars cannot be a legitimate Zoom validation token. + let long_token = "a".repeat(129); + let payload = format!( + r#"{{"event":"endpoint.url_validation","event_ts":1234567890,"payload":{{"plainToken":"{}"}}}}"#, + long_token + ); + + let handler = WebhookType::Zoom.get_webhook_handler().unwrap(); + let config_data = SignatureConfigData { secret_key: "zoom_secret" }; + let result = handler.handle_challenge_request(&HeaderMap::new(), &config_data, &payload); + assert!(matches!( + result, + Err(AuthenticationError::InvalidChallengeResponse(_)) + )); + } + // --- Custom webhook end-to-end --- #[test] From 2bff250f89beeb06025bd6edca478492695f8d42 Mon Sep 17 00:00:00 2001 From: Guilhem Date: Tue, 2 Jun 2026 14:58:21 +0200 Subject: [PATCH 16/61] feat(frontend): harmonize diff button placement in script and raw app editors (#9410) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(frontend): harmonize diff button placement across editors Co-Authored-By: Claude Opus 4.8 (1M context) * refactor(frontend): address review nits — drop unused diffDrawer param, fix stale comments Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Claude Opus 4.8 (1M context) --- .../src/lib/components/ScriptBuilder.svelte | 77 +++++++++++-------- .../raw_apps/RawAppEditorHeader.svelte | 62 ++++++++------- .../sessions/ScriptEditorView.svelte | 2 +- 3 files changed, 79 insertions(+), 62 deletions(-) diff --git a/frontend/src/lib/components/ScriptBuilder.svelte b/frontend/src/lib/components/ScriptBuilder.svelte index 9dc36e3503..8f4e75ff3d 100644 --- a/frontend/src/lib/components/ScriptBuilder.svelte +++ b/frontend/src/lib/components/ScriptBuilder.svelte @@ -55,6 +55,7 @@ Bug, CheckCircle, Code, + DiffIcon, EllipsisVertical, Plus, Rocket, @@ -101,7 +102,6 @@ import DraftTriggersConfirmationModal from './common/confirmationModal/DraftTriggersConfirmationModal.svelte' import { Triggers } from './triggers/triggers.svelte' import type { ScriptBuilderProps } from './script_builder' - import type { DiffDrawerI } from './diff_drawer' import WorkerTagSelect from './WorkerTagSelect.svelte' import type { ButtonType } from './common/button/model' import DebounceLimit from './flows/DebounceLimit.svelte' @@ -804,13 +804,34 @@ // Inside an AI session pane (which injects an aiChatManager via context) the // extra deploy-dropdown options — Deploy & Stay here, Fork, Edit in workspace // fork, Exit & See details, Export — don't make sense: the session always - // stays put and is already scoped to a fork. Only "Show diff" is kept. + // stays put and is already scoped to a fork. Diff is exposed as a standalone + // top-bar button (rendered independently of the session pane), not here. const inSessionPane = !!getContext('aiChatManager') + async function openDiffDrawer() { + if (!savedScript) { + return + } + await syncWithDeployed() + + const currentDraftTriggers = structuredClone(triggersState.getDraftTriggersSnapshot()) + + const deployed = deployedValue ?? savedScript + const current = { ...script, draft_triggers: currentDraftTriggers } + if (current.assets && !current.assets.length) delete current.assets + + diffDrawer?.openDrawer() + diffDrawer?.setDiff({ + mode: 'normal', + deployed, + draft: savedScript['draft'], + current + }) + } + function computeDropdownItems( initialPath: string, - savedScript: NewScriptWithDraftAndDraftTriggers | undefined, - diffDrawer: DiffDrawerI | undefined + savedScript: NewScriptWithDraftAndDraftTriggers | undefined ) { let dropdownItems: { label: string; onClick: () => void }[] = initialPath != '' && customUi?.topBar?.extraDeployOptions != false @@ -841,35 +862,6 @@ : []) ] : []), - ...(customUi?.topBar?.diff !== false && savedScript && diffDrawer - ? [ - { - label: 'Show diff', - onClick: async () => { - if (!savedScript) { - return - } - await syncWithDeployed() - - const currentDraftTriggers = structuredClone( - triggersState.getDraftTriggersSnapshot() - ) - - const deployed = deployedValue ?? savedScript - const current = { ...script, draft_triggers: currentDraftTriggers } - if (current.assets && !current.assets.length) delete current.assets - - diffDrawer?.openDrawer() - diffDrawer?.setDiff({ - mode: 'normal', - deployed, - draft: savedScript['draft'], - current - }) - } - } - ] - : []), ...(!inSessionPane && !script.draft_only && script.kind === 'script' && @@ -2035,6 +2027,21 @@ {/if} {/snippet} + {#snippet diffButton()} + {#if customUi?.topBar?.diff != false} + + {/if} + {/snippet} {#if compactTopbar} {#snippet buttonReplacement()} @@ -2048,8 +2055,10 @@ /> {/snippet} + {@render diffButton()} {@render settingsButton()} {:else} + {@render diffButton()} {#if customUi?.topBar?.tagEdit != false} {#if $workerTags} {#if $workerTags?.length ?? 0 > 0} @@ -2080,7 +2089,7 @@ handleEditScript(false, detail)} />
diff --git a/frontend/src/lib/components/raw_apps/RawAppEditorHeader.svelte b/frontend/src/lib/components/raw_apps/RawAppEditorHeader.svelte index 7f9e8e9807..dedb7b5d11 100644 --- a/frontend/src/lib/components/raw_apps/RawAppEditorHeader.svelte +++ b/frontend/src/lib/components/raw_apps/RawAppEditorHeader.svelte @@ -364,6 +364,29 @@ }) } + async function openDiffDrawer() { + if (!savedApp) { + return + } + + // deployedValue should be syncronized when we open Diff + await syncWithDeployed() + + diffDrawer?.openDrawer() + diffDrawer?.setDiff({ + mode: 'normal', + deployed: deployedValue ?? savedApp, + draft: savedApp.draft, + current: { + summary: summary, + value: app, + path: newEditedPath || savedApp.draft?.path || savedApp.path, + policy, + custom_path: customPath + } + }) + } + async function updateApp(npath: string) { if (!app) { sendUserToast(`App hasn't been loaded yet`, true) @@ -682,33 +705,6 @@ action: () => { publishToHubDrawerOpen = true } - }, - { - displayName: 'Diff', - icon: DiffIcon, - action: async () => { - if (!savedApp) { - return - } - - // deployedValue should be syncronized when we open Diff - await syncWithDeployed() - - diffDrawer?.openDrawer() - diffDrawer?.setDiff({ - mode: 'normal', - deployed: deployedValue ?? savedApp, - draft: savedApp.draft, - current: { - summary: summary, - value: app, - path: newEditedPath || savedApp.draft?.path || savedApp.path, - policy, - custom_path: customPath - } - }) - }, - disabled: !savedApp } ]) @@ -965,6 +961,18 @@ {/snippet} + +
{:else} diff --git a/frontend/src/lib/components/DisplayResultControlBar.svelte b/frontend/src/lib/components/DisplayResultControlBar.svelte index d45ca3f92d..45011d2db1 100644 --- a/frontend/src/lib/components/DisplayResultControlBar.svelte +++ b/frontend/src/lib/components/DisplayResultControlBar.svelte @@ -4,6 +4,7 @@ import Popover from './Popover.svelte' import { copyToClipboard } from '$lib/utils' import { downloadViaClient, shouldDownloadViaClient } from '$lib/utils/downloadFile' + import { appendViewToken } from '$lib/viewToken' import type { DisplayResultUi } from './custom_ui' import { createEventDispatcher } from 'svelte' @@ -41,9 +42,11 @@ let resultApiPath = $derived( workspaceId && jobId - ? nodeId - ? `/w/${workspaceId}/jobs/result_by_id/${jobId}/${nodeId}` - : `/w/${workspaceId}/jobs_u/completed/get_result/${jobId}` + ? appendViewToken( + nodeId + ? `/w/${workspaceId}/jobs/result_by_id/${jobId}/${nodeId}` + : `/w/${workspaceId}/jobs_u/completed/get_result/${jobId}` + ) : undefined ) let downloadName = $derived(`${filename ?? 'result'}.json`) diff --git a/frontend/src/lib/components/FlowStatusViewerInner.svelte b/frontend/src/lib/components/FlowStatusViewerInner.svelte index 809eff58d9..d0bf66d61b 100644 --- a/frontend/src/lib/components/FlowStatusViewerInner.svelte +++ b/frontend/src/lib/components/FlowStatusViewerInner.svelte @@ -28,6 +28,7 @@ import ModuleStatus from './ModuleStatus.svelte' import { clone, isScriptPreview, msToSec, readFieldsRecursively, truncateRev } from '$lib/utils' import { downloadViaClient, shouldDownloadViaClient } from '$lib/utils/downloadFile' + import { appendViewToken } from '$lib/viewToken' import JobArgs from './JobArgs.svelte' import { ChevronDown, Download, ExternalLink, Hourglass } from 'lucide-svelte' import { deepEqual } from 'fast-equals' @@ -1839,7 +1840,9 @@ style="min-height: {minTabHeight}px" > {#if !hideDownloadLogs && !isReplay && job?.id} - {@const logsApiPath = `/w/${workspace}/jobs_u/get_flow_all_logs/${job.id}`} + {@const logsApiPath = appendViewToken( + `/w/${workspace}/jobs_u/get_flow_all_logs/${job.id}` + )} {@const logsName = `windmill_flow_logs_${job.id}.txt`}
{#if shouldDownloadViaClient()} diff --git a/frontend/src/lib/components/JobArgs.svelte b/frontend/src/lib/components/JobArgs.svelte index 77c94de11e..ffa90bb941 100644 --- a/frontend/src/lib/components/JobArgs.svelte +++ b/frontend/src/lib/components/JobArgs.svelte @@ -14,6 +14,7 @@ import { deepEqual } from 'fast-equals' import { isWindmillTooBigObject } from './job_args' import { downloadViaClient, shouldDownloadViaClient } from '$lib/utils/downloadFile' + import { appendViewToken } from '$lib/viewToken' interface Props { id?: string | undefined @@ -29,7 +30,9 @@ let jsonStr = $state('') const argsDownloadName = 'windmill-args.json' - let argsApiPath = $derived(id && workspace ? `/w/${workspace}/jobs_u/get_args/${id}` : undefined) + let argsApiPath = $derived( + id && workspace ? appendViewToken(`/w/${workspace}/jobs_u/get_args/${id}`) : undefined + ) let argsDataHref = $derived(`data:text/json;charset=utf-8,${encodeURIComponent(jsonStr)}`) function pythonCode() { diff --git a/frontend/src/lib/components/JobLoader.svelte b/frontend/src/lib/components/JobLoader.svelte index 7913fd4be7..1aa24e68a5 100644 --- a/frontend/src/lib/components/JobLoader.svelte +++ b/frontend/src/lib/components/JobLoader.svelte @@ -15,6 +15,7 @@ type OpenFlow } from '$lib/gen' import { workspaceStore } from '$lib/stores' + import { getViewToken } from '$lib/viewToken' import { WM_LOGS_SKIPPED } from '$lib/consts' import { getContext, onDestroy, tick, untrack } from 'svelte' import type { SupportedLanguage } from '$lib/common' @@ -47,6 +48,9 @@ noLogs?: boolean workspaceOverride?: string | undefined notfound?: boolean + /** Status/body of the last load failure, so callers can distinguish e.g. a + * 403 (job exists but no access — offer a share link) from a 404. */ + loadError?: { status?: number; message?: string } | undefined allowConcurentRequests?: boolean jobUpdateLastFetch?: Date | undefined toastError?: boolean @@ -65,6 +69,7 @@ allowConcurentRequests = false, workspaceOverride = undefined, notfound = $bindable(false), + loadError = $bindable(undefined), jobUpdateLastFetch = $bindable(undefined), toastError = false, onlyResult = false, @@ -600,9 +605,14 @@ } } notfound = false + loadError = undefined } catch (err) { + const status = (err as any)?.status + loadError = { status, message: (err as any)?.body ?? (err as any)?.message } errorIteration += 1 - if (errorIteration == 5) { + // Auth failures won't resolve by retrying: surface them immediately so + // the caller can show the right message (e.g. 403 -> request a share link). + if (status === 403 || status === 404 || errorIteration == 5) { notfound = true job = undefined clearCurrentId() @@ -754,6 +764,13 @@ params.set('token', token.token) } + // Share read link: SSE/EventSource can't set the X-View-Token header, + // so carry the token as a query param instead. + const viewToken = getViewToken() + if (viewToken) { + params.set('view_token', viewToken) + } + const sseUrl = `/api/w/${workspace}/jobs_u/getupdate_sse/${id}?${params.toString()}` currentEventSource = new EventSource(sseUrl) diff --git a/frontend/src/lib/components/LogViewer.svelte b/frontend/src/lib/components/LogViewer.svelte index ca90e3968f..b5a31265be 100644 --- a/frontend/src/lib/components/LogViewer.svelte +++ b/frontend/src/lib/components/LogViewer.svelte @@ -17,6 +17,7 @@ import { base } from '$lib/base' import { withExternalDomain } from '$lib/externalDomain' import { downloadViaClient, shouldDownloadViaClient } from '$lib/utils/downloadFile' + import { appendViewToken } from '$lib/viewToken' import { workspaceStore } from '$lib/stores' import { AnsiUp } from 'ansi_up' import NoWorkerWithTagWarning from './runs/NoWorkerWithTagWarning.svelte' @@ -241,7 +242,7 @@ fetchedSkippedJobId = undefined } }) - let logsApiPath = $derived(`/w/${$workspaceStore}/jobs_u/get_logs/${jobId}`) + let logsApiPath = $derived(appendViewToken(`/w/${$workspaceStore}/jobs_u/get_logs/${jobId}`)) let downloadHref = $derived(withExternalDomain(`${base}/api${logsApiPath}`)) let downloadName = $derived(`windmill_logs_${jobId}.txt`) let truncatedContent = $derived( diff --git a/frontend/src/lib/viewToken.ts b/frontend/src/lib/viewToken.ts new file mode 100644 index 0000000000..18c21d27d8 --- /dev/null +++ b/frontend/src/lib/viewToken.ts @@ -0,0 +1,44 @@ +import { OpenAPI } from '$lib/gen' + +/** + * Share-read-link support. When viewing a run via a share link + * (`/run/{id}?view_token=...`), the token grants the current authenticated member + * read access to that job and its flow subtree on the backend. + * + * The token is attached to every generated-client request via the `X-View-Token` + * header (registered once below) so we don't have to thread it through every + * `JobService` call. `EventSource`/SSE can't set headers, so those URLs read + * `getViewToken()` and append it as a `view_token` query param instead. + */ +let currentViewToken: string | undefined = undefined + +export function setViewToken(token: string | undefined): void { + currentViewToken = token || undefined +} + +export function getViewToken(): string | undefined { + return currentViewToken +} + +/** + * Append the current view token as a `view_token` query param to a URL/path. + * Used for download links (plain `` and `downloadViaClient`), which don't + * go through the request interceptor that adds the `X-View-Token` header. + * Returns the url unchanged when no share link is active. + */ +export function appendViewToken(url: string): string { + if (!currentViewToken) return url + const sep = url.includes('?') ? '&' : '?' + return `${url}${sep}view_token=${encodeURIComponent(currentViewToken)}` +} + +// Register the request interceptor exactly once. It is a no-op unless a view token +// is currently set, so it is safe to keep installed for the whole session. +OpenAPI.interceptors.request.use((options) => { + if (currentViewToken) { + const headers = new Headers(options.headers) + headers.set('X-View-Token', currentViewToken) + options.headers = headers + } + return options +}) diff --git a/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte b/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte index 05b53ece63..1357038c39 100644 --- a/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte @@ -36,7 +36,8 @@ ClipboardCopy, GitBranch, GitFork, - EllipsisVertical + EllipsisVertical, + Share2 } from 'lucide-svelte' import DisplayResult from '$lib/components/DisplayResult.svelte' @@ -85,6 +86,7 @@ } from '$lib/components/flows/FlowAssetsHandler.svelte' import JobAssetsViewer from '$lib/components/assets/JobAssetsViewer.svelte' import { page } from '$app/state' + import { setViewToken } from '$lib/viewToken' import { twMerge } from 'tailwind-merge' import FlowRestartButton from '$lib/components/FlowRestartButton.svelte' import { useNestedRestartState } from '$lib/components/useNestedRestartState.svelte' @@ -120,6 +122,7 @@ let testIsLoading = $state(false) let jobLoader: JobLoader | undefined = $state(undefined) + let loadError: { status?: number; message?: string } | undefined = $state(undefined) // Flow execution status state let suspendStatus: import('$lib/utils').StateStore> = @@ -146,6 +149,34 @@ concurrencyKey = await ConcurrencyGroupsService.getConcurrencyKey({ id: job.id }) } + // Share read link: if the URL carries a `view_token`, install it so every job + // read on this page (incl. flow steps, args, logs, SSE) is authorized by it. + // Set eagerly at init (before JobLoader mounts and fires its first fetch), and + // reactively keep it in sync across client-side navigation. + setViewToken(page.url.searchParams.get('view_token') ?? undefined) + $effect(() => { + setViewToken(page.url.searchParams.get('view_token') ?? undefined) + }) + onDestroy(() => setViewToken(undefined)) + + async function shareReadLink(id: string): Promise { + try { + const workspace = $workspaceStore! + const token = (await JobService.getJobViewToken({ workspace, id })).trim() + // Pin the workspace in the link: the token is signed with this workspace's + // key, and the logged layout only switches `$workspaceStore` when the URL + // carries `workspace=`. Without it a recipient whose active workspace + // differs would open the run (and validate the token) against the wrong one. + const url = `${window.location.origin}${base}/run/${id}?workspace=${encodeURIComponent( + workspace + )}&view_token=${encodeURIComponent(token)}` + copyToClipboard(url) + sendUserToast('Read-only share link copied to clipboard') + } catch (e) { + sendUserToast(`Failed to create share link: ${e}`, true) + } + } + async function deleteCompletedJob(id: string): Promise { await JobService.deleteCompletedJob({ workspace: $workspaceStore!, id }) getJob() @@ -447,6 +478,7 @@ bind:jobUpdateLastFetch workspaceOverride={$workspaceStore} bind:notfound + bind:loadError /> {/if} @@ -454,7 +486,28 @@ -{#if notfound || (job?.workspace_id != undefined && $workspaceStore != undefined && job?.workspace_id != $workspaceStore)} +{#if loadError?.status === 403} +
+
+ +
+

+ This run exists in {$workspaceStore}, but you don't + have permission to view it. +

+

+ Ask a colleague who can see it to open the run and use the + Share button to send you a read-only link. Opening that + link will grant you access to this run (and its steps). +

+
+ +
+ +
+
+
+{:else if notfound || (job?.workspace_id != undefined && $workspaceStore != undefined && job?.workspace_id != $workspaceStore)}

{/if} {/if} + {#if job} + + {/if} {@const stem = job?.job_kind === 'script_hub' ? '/scripts' : `/${job?.job_kind}s`} {@const viewHref = `${stem}/get/${isScript ? job?.script_hash : job?.script_path}`} {#if (job?.job_kind == 'flow' || isFlowPreview(job?.job_kind)) && job?.['running'] && job?.parent_job == undefined} From 7edf3f02122e20fde1e95e0252e7bda641075326 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 3 Jun 2026 10:34:43 +0200 Subject: [PATCH 20/61] fix(auth): filter script/flow listings by token scope (GHSA-2ppx-66jv-wpw5) (#9426) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A token scoped to a single script or flow path (e.g. `scripts:read:f/allowed/*`) could call `GET .../scripts/list_search` (or `/list`) and receive `path` + full `content` for every script the underlying user could see — likewise `flows/list_search` leaked the full flow `value`. Route-level scope checks only validate `domain:action`, and the listing handlers did no per-row scope filtering, leaking out-of-scope source/definitions to narrowly-scoped tokens. Apply `build_scope_path_predicate` (added in #9302 for resources/variables) to `list_search_scripts`, `list_scripts`, `list_search_flows`, and `list_flows`, mirroring the resources/variables fix exactly. Unscoped tokens and tokens whose only scopes are `if_jobs:filter_tags:*` are unaffected. Adds integration regression tests (scripts + flows) covering: path-scoped token sees only in-scope paths, broad `*:read` token still sees all RLS-visible items, tag-filter-only and unscoped tokens unchanged. Co-authored-by: Claude Opus 4.8 (1M context) --- backend/windmill-api-flows/src/flows.rs | 13 +- .../tests/flows.rs | 115 ++++++++++++++++-- .../tests/scripts.rs | 104 ++++++++++++++++ backend/windmill-api-scripts/src/scripts.rs | 11 +- 4 files changed, 226 insertions(+), 17 deletions(-) diff --git a/backend/windmill-api-flows/src/flows.rs b/backend/windmill-api-flows/src/flows.rs index 546bda32f0..c63d53171c 100644 --- a/backend/windmill-api-flows/src/flows.rs +++ b/backend/windmill-api-flows/src/flows.rs @@ -16,7 +16,8 @@ use axum::{ }; use windmill_api_auth::{ auth::{list_tokens_internal, TruncatedTokenWithEmail}, - check_scopes, maybe_refresh_folders, require_owner_of_path, ApiAuthed, + build_scope_path_predicate, check_scopes, maybe_refresh_folders, require_owner_of_path, + ApiAuthed, }; use windmill_common::workspaces::{check_deploy_rules, RuleCheckResult}; use windmill_common::{ @@ -108,9 +109,10 @@ async fn list_search_flows( let n = 3; let mut tx = user_db.begin(&authed).await?; + let allowed = build_scope_path_predicate(&authed, "flows", "read"); let rows = sqlx::query_as::<_, SearchFlow>( "SELECT flow.path, flow_version.value - FROM flow + FROM flow LEFT JOIN flow_version ON flow_version.id = flow.versions[array_upper(flow.versions, 1)] WHERE flow.workspace_id = $1 LIMIT $2", ) @@ -119,6 +121,7 @@ async fn list_search_flows( .fetch_all(&mut *tx) .await? .into_iter() + .filter(|r| allowed(&r.path)) .collect::>(); tx.commit().await?; Ok(Json(rows)) @@ -212,9 +215,13 @@ async fn list_flows( let sql = sqlb.sql().map_err(|e| Error::internal_err(e.to_string()))?; let mut tx = user_db.begin(&authed).await?; + let allowed = build_scope_path_predicate(&authed, "flows", "read"); let rows = sqlx::query_as::<_, ListableFlow>(&sql) .fetch_all(&mut *tx) - .await?; + .await? + .into_iter() + .filter(|r| allowed(&r.path)) + .collect::>(); tx.commit().await?; Ok(Json(rows)) } diff --git a/backend/windmill-api-integration-tests/tests/flows.rs b/backend/windmill-api-integration-tests/tests/flows.rs index ff3f86bf2d..b6075c8e69 100644 --- a/backend/windmill-api-integration-tests/tests/flows.rs +++ b/backend/windmill-api-integration-tests/tests/flows.rs @@ -259,12 +259,10 @@ async fn test_flow_endpoints(db: Pool) -> anyhow::Result<()> { // ===== Hub endpoints (require external network, expect 500 or 200) ===== // --- hub/list --- - let resp = authed(client().get(format!( - "http://localhost:{port}/api/flows/hub/list" - ))) - .send() - .await - .unwrap(); + let resp = authed(client().get(format!("http://localhost:{port}/api/flows/hub/list"))) + .send() + .await + .unwrap(); assert!( resp.status() == 200 || resp.status() == 500, "hub/list: unexpected status {}", @@ -272,12 +270,10 @@ async fn test_flow_endpoints(db: Pool) -> anyhow::Result<()> { ); // --- hub/get --- - let resp = authed(client().get(format!( - "http://localhost:{port}/api/flows/hub/get/1" - ))) - .send() - .await - .unwrap(); + let resp = authed(client().get(format!("http://localhost:{port}/api/flows/hub/get/1"))) + .send() + .await + .unwrap(); assert!( resp.status() == 200 || resp.status() == 500, "hub/get: unexpected status {}", @@ -286,3 +282,98 @@ async fn test_flow_endpoints(db: Pool) -> anyhow::Result<()> { Ok(()) } + +/// Regression test for GHSA-2ppx-66jv-wpw5: a path-scoped token must only see +/// the flows within its scope when listing, even though the route-level scope +/// check only validates `domain:action`. Before the fix, `list_search` returned +/// `path` + the full flow `value` for every flow the underlying user could see, +/// leaking out-of-scope flow definitions to narrowly-scoped tokens. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_list_search_scope_filtering(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/flows"); + + // Create two folders and one flow in each, as the (super-admin) test user. + for folder in ["allowed", "private"] { + let resp = authed(client().post(format!( + "http://localhost:{port}/api/w/test-workspace/folders/create" + ))) + .json(&json!({ "name": folder })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "create folder: {}", resp.text().await?); + } + + for path in ["f/allowed/foo", "f/private/bar"] { + let resp = authed(client().post(format!("{base}/create"))) + .json(&new_flow(path, "summary")) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 201, "create {path}: {}", resp.text().await?); + } + + // Helper: GET /list_search with an arbitrary bearer token, returning the set + // of flow paths visible to that token. + async fn list_search_paths(port: u16, token: &str) -> Vec { + let resp = client() + .get(format!( + "http://localhost:{port}/api/w/test-workspace/flows/list_search" + )) + .header("Authorization", format!("Bearer {token}")) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + resp.json::>() + .await + .unwrap() + .into_iter() + .map(|s| s["path"].as_str().unwrap().to_string()) + .collect() + } + + // Insert three tokens for the same super-admin user, differing only by scope. + sqlx::query( + "INSERT INTO token (token_hash, token_prefix, token, email, label, super_admin, scopes) VALUES + (encode(sha256('SCOPED_TOKEN'::bytea), 'hex'), 'SCOPED_TOK', 'SCOPED_TOKEN', 'test@windmill.dev', 'scoped', true, ARRAY['flows:read:f/allowed/*']), + (encode(sha256('BROAD_TOKEN'::bytea), 'hex'), 'BROAD_TOK', 'BROAD_TOKEN', 'test@windmill.dev', 'broad', true, ARRAY['flows:read']), + (encode(sha256('TAG_TOKEN'::bytea), 'hex'), 'TAG_TOK', 'TAG_TOKEN', 'test@windmill.dev', 'tag-only', true, ARRAY['if_jobs:filter_tags:default'])", + ) + .execute(&db) + .await?; + + // Path-scoped token: only sees flows within `f/allowed/*`. + let scoped = list_search_paths(port, "SCOPED_TOKEN").await; + assert!( + scoped.contains(&"f/allowed/foo".to_string()), + "scoped token should see f/allowed/foo, got: {scoped:?}" + ); + assert!( + !scoped.contains(&"f/private/bar".to_string()), + "scoped token must NOT see f/private/bar, got: {scoped:?}" + ); + + // Broad `flows:read` token: still sees every RLS-visible flow. + let broad = list_search_paths(port, "BROAD_TOKEN").await; + assert!(broad.contains(&"f/allowed/foo".to_string())); + assert!( + broad.contains(&"f/private/bar".to_string()), + "broad flows:read token should see all flows, got: {broad:?}" + ); + + // Tag-filter-only token is not scope-restricted: unchanged, sees all. + let tag_only = list_search_paths(port, "TAG_TOKEN").await; + assert!(tag_only.contains(&"f/allowed/foo".to_string())); + assert!(tag_only.contains(&"f/private/bar".to_string())); + + // Unscoped token (no scopes column set): unchanged, sees all. + let unscoped = list_search_paths(port, "SECRET_TOKEN").await; + assert!(unscoped.contains(&"f/allowed/foo".to_string())); + assert!(unscoped.contains(&"f/private/bar".to_string())); + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/scripts.rs b/backend/windmill-api-integration-tests/tests/scripts.rs index f5e78f880f..c374b757a4 100644 --- a/backend/windmill-api-integration-tests/tests/scripts.rs +++ b/backend/windmill-api-integration-tests/tests/scripts.rs @@ -463,3 +463,107 @@ async fn test_auto_parent_resolves_parent_hash(db: Pool) -> anyhow::Re Ok(()) } + +/// Regression test for GHSA-2ppx-66jv-wpw5: a path-scoped token must only see +/// the scripts within its scope when listing, even though the route-level scope +/// check only validates `domain:action`. Before the fix, `list_search` (and +/// `list`) returned `path` + full `content` for every script the underlying +/// user could see, leaking out-of-scope script source to narrowly-scoped tokens. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_list_search_scope_filtering(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/scripts"); + + // Create two folders and one script in each, as the (super-admin) test user. + for folder in ["allowed", "private"] { + let resp = authed(client().post(format!( + "http://localhost:{port}/api/w/test-workspace/folders/create" + ))) + .json(&json!({ "name": folder })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "create folder: {}", resp.text().await?); + } + + for (path, content) in [ + ( + "f/allowed/foo", + "export async function main() { return 'allowed'; }", + ), + ( + "f/private/bar", + "export async function main() { return 'secret'; }", + ), + ] { + let resp = authed(client().post(format!("{base}/create"))) + .json(&new_script(path, "summary", content)) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 201, "create {path}: {}", resp.text().await?); + } + + // Helper: GET /list_search with an arbitrary bearer token, returning the set + // of script paths visible to that token. + async fn list_search_paths(port: u16, token: &str) -> Vec { + let resp = client() + .get(format!( + "http://localhost:{port}/api/w/test-workspace/scripts/list_search" + )) + .header("Authorization", format!("Bearer {token}")) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + resp.json::>() + .await + .unwrap() + .into_iter() + .map(|s| s["path"].as_str().unwrap().to_string()) + .collect() + } + + // Insert three tokens for the same super-admin user, differing only by scope. + sqlx::query( + "INSERT INTO token (token_hash, token_prefix, token, email, label, super_admin, scopes) VALUES + (encode(sha256('SCOPED_TOKEN'::bytea), 'hex'), 'SCOPED_TOK', 'SCOPED_TOKEN', 'test@windmill.dev', 'scoped', true, ARRAY['scripts:read:f/allowed/*']), + (encode(sha256('BROAD_TOKEN'::bytea), 'hex'), 'BROAD_TOK', 'BROAD_TOKEN', 'test@windmill.dev', 'broad', true, ARRAY['scripts:read']), + (encode(sha256('TAG_TOKEN'::bytea), 'hex'), 'TAG_TOK', 'TAG_TOKEN', 'test@windmill.dev', 'tag-only', true, ARRAY['if_jobs:filter_tags:default'])", + ) + .execute(&db) + .await?; + + // Path-scoped token: only sees scripts within `f/allowed/*`. + let scoped = list_search_paths(port, "SCOPED_TOKEN").await; + assert!( + scoped.contains(&"f/allowed/foo".to_string()), + "scoped token should see f/allowed/foo, got: {scoped:?}" + ); + assert!( + !scoped.contains(&"f/private/bar".to_string()), + "scoped token must NOT see f/private/bar, got: {scoped:?}" + ); + + // Broad `scripts:read` token: still sees every RLS-visible script. + let broad = list_search_paths(port, "BROAD_TOKEN").await; + assert!(broad.contains(&"f/allowed/foo".to_string())); + assert!( + broad.contains(&"f/private/bar".to_string()), + "broad scripts:read token should see all scripts, got: {broad:?}" + ); + + // Tag-filter-only token is not scope-restricted: unchanged, sees all. + let tag_only = list_search_paths(port, "TAG_TOKEN").await; + assert!(tag_only.contains(&"f/allowed/foo".to_string())); + assert!(tag_only.contains(&"f/private/bar".to_string())); + + // Unscoped token (no scopes column set): unchanged, sees all. + let unscoped = list_search_paths(port, "SECRET_TOKEN").await; + assert!(unscoped.contains(&"f/allowed/foo".to_string())); + assert!(unscoped.contains(&"f/private/bar".to_string())); + + Ok(()) +} diff --git a/backend/windmill-api-scripts/src/scripts.rs b/backend/windmill-api-scripts/src/scripts.rs index 0dc86bfda8..c783f18d97 100644 --- a/backend/windmill-api-scripts/src/scripts.rs +++ b/backend/windmill-api-scripts/src/scripts.rs @@ -9,7 +9,8 @@ use axum::extract::Multipart; use windmill_api_auth::{ auth::{list_tokens_internal, AuthCache, TruncatedTokenWithEmail}, - check_scopes, maybe_refresh_folders, require_owner_of_path, ApiAuthed, + build_scope_path_predicate, check_scopes, maybe_refresh_folders, require_owner_of_path, + ApiAuthed, }; use windmill_common::{ utils::{BulkDeleteRequest, WithStarredInfoQuery, HTTP_CLIENT}, @@ -275,6 +276,7 @@ async fn list_search_scripts( #[cfg(not(feature = "enterprise"))] let n = 10; + let allowed = build_scope_path_predicate(&authed, "scripts", "read"); let rows = sqlx::query_as!( SearchScript, "SELECT path, content from script WHERE workspace_id = $1 AND archived = false LIMIT $2", @@ -284,6 +286,7 @@ async fn list_search_scripts( .fetch_all(&mut *tx) .await? .into_iter() + .filter(|r| allowed(&r.path)) .collect::>(); tx.commit().await?; Ok(Json(rows)) @@ -438,9 +441,13 @@ async fn list_scripts( let sql = sqlb.sql().map_err(|e| Error::internal_err(e.to_string()))?; let mut tx = user_db.begin(&authed).await?; + let allowed = build_scope_path_predicate(&authed, "scripts", "read"); let rows = sqlx::query_as::<_, ListableScript>(&sql) .fetch_all(&mut *tx) - .await?; + .await? + .into_iter() + .filter(|r| allowed(&r.path)) + .collect::>(); tx.commit().await?; Ok(Json(rows)) } From 3b2e748daf0a8ec4447c30423068df803f3f9ca2 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 3 Jun 2026 10:42:39 +0200 Subject: [PATCH 21/61] feat(frontend): add rebuild dependency map button to workspace settings (#9424) Co-authored-by: Claude Opus 4.8 (1M context) --- .../WorkspaceDependenciesSettings.svelte | 42 ++++++++++++++++++- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/frontend/src/lib/components/workspaceSettings/WorkspaceDependenciesSettings.svelte b/frontend/src/lib/components/workspaceSettings/WorkspaceDependenciesSettings.svelte index b2e95f39c3..b5ab5f5124 100644 --- a/frontend/src/lib/components/workspaceSettings/WorkspaceDependenciesSettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/WorkspaceDependenciesSettings.svelte @@ -12,7 +12,7 @@ import DrawerContent from '$lib/components/common/drawer/DrawerContent.svelte' import HighlightCode from '$lib/components/HighlightCode.svelte' import { workspaceStore, userStore } from '$lib/stores' - import { Plus, FileText, Search, Code2, Edit, Eye } from 'lucide-svelte' + import { Plus, FileText, Search, Code2, Edit, Eye, RefreshCw } from 'lucide-svelte' import { WorkspaceDependenciesService, WorkspaceService } from '$lib/gen' import type { WorkspaceDependencies, ScriptLang } from '$lib/gen' import { untrack } from 'svelte' @@ -24,6 +24,7 @@ let workspaceDependencies: WorkspaceDependencies[] | undefined = $state() let filteredItems: (WorkspaceDependencies & { marked?: string })[] | undefined = $state() let workspaceDependenciesEditor: WorkspaceDependenciesEditor | undefined = $state() + let rebuildingDependencyMap = $state(false) // View modal state let viewDrawer: Drawer | undefined = $state() @@ -78,6 +79,20 @@ } }) + async function rebuildDependencyMap(): Promise { + if (!$workspaceStore) return + rebuildingDependencyMap = true + try { + const status = await WorkspaceService.rebuildDependencyMap({ workspace: $workspaceStore }) + sendUserToast(status) + } catch (error) { + console.error('Error rebuilding dependency map:', error) + sendUserToast(`Failed to rebuild dependency map: ${error.message}`, true) + } finally { + rebuildingDependencyMap = false + } + } + async function createNewWorkspaceDependencies() { await workspaceDependenciesEditor?.initNew() } @@ -270,7 +285,7 @@

-
+
{#if !filteredItems} {#each new Array(3) as _} @@ -411,6 +426,29 @@ {/if}
+{#if $userStore?.is_admin || $userStore?.is_super_admin} +
+
+ Rebuild dependency map + + Rebuilds the workspace dependency map from scratch. This should almost never be needed — + only if dependency tracking has gotten out of sync, e.g. after orphaned references are + reported in the logs. + +
+ +
+{/if} + {#snippet actions()} From 7031744a199f0bf8b8e35043afa959977e5ecdbd Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 3 Jun 2026 10:47:19 +0200 Subject: [PATCH 22/61] fix(nsjail): precompile python stdlib + raise download rlimit_as (#9429) Co-authored-by: Claude Opus 4.8 (1M context) --- .github/DockerfileBackendTests | 2 +- .github/workflows/backend-test-windows.yml | 2 +- .github/workflows/backend-test.yml | 2 +- Dockerfile | 11 +++++++---- .../windmill-worker/nsjail/download.py.config.proto | 9 ++++++++- backend/windmill-worker/src/python_versions.rs | 5 +++++ docker/DockerfileSlim | 9 ++++++--- docker/DockerfileSlimEe | 9 ++++++--- 8 files changed, 35 insertions(+), 14 deletions(-) diff --git a/.github/DockerfileBackendTests b/.github/DockerfileBackendTests index 4473c00f0a..88275f204b 100644 --- a/.github/DockerfileBackendTests +++ b/.github/DockerfileBackendTests @@ -28,7 +28,7 @@ ENV PATH="${PATH}:/usr/local/go/bin" ENV GO_PATH=/usr/local/go/bin/go # UV -RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.9.24/uv-installer.sh | sh && mv /usr/local/cargo/bin/uv /usr/local/bin/uv +RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.9.25/uv-installer.sh | sh && mv /usr/local/cargo/bin/uv /usr/local/bin/uv ENV TZ=Etc/UTC diff --git a/.github/workflows/backend-test-windows.yml b/.github/workflows/backend-test-windows.yml index f7c49654d1..1c73e5d429 100644 --- a/.github/workflows/backend-test-windows.yml +++ b/.github/workflows/backend-test-windows.yml @@ -74,7 +74,7 @@ jobs: - uses: astral-sh/setup-uv@v6.2.1 with: - version: "0.9.24" + version: "0.9.25" - uses: shivammathur/setup-php@v2 with: diff --git a/.github/workflows/backend-test.yml b/.github/workflows/backend-test.yml index 9009b47e9d..8f1f15447c 100644 --- a/.github/workflows/backend-test.yml +++ b/.github/workflows/backend-test.yml @@ -62,7 +62,7 @@ jobs: node-version: "20" - uses: astral-sh/setup-uv@v6.2.1 with: - version: "0.9.24" + version: "0.9.25" - uses: shivammathur/setup-php@v2 with: php-version: "8.3" diff --git a/Dockerfile b/Dockerfile index 14aa5363ef..2c55cf36f2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -233,11 +233,14 @@ ENV PATH="${PATH}:/usr/local/go/bin" ENV GO_PATH=/usr/local/go/bin/go # Install UV -RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.9.24/uv-installer.sh | sh && mv /root/.local/bin/uv /usr/local/bin/uv +RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.9.25/uv-installer.sh | sh && mv /root/.local/bin/uv /usr/local/bin/uv # Preinstall python runtimes to temp build location (will copy with world-writable perms later) -RUN UV_CACHE_DIR=/tmp/build_cache/uv UV_PYTHON_INSTALL_DIR=/tmp/build_cache/py_runtime uv python install 3.11 -RUN UV_CACHE_DIR=/tmp/build_cache/uv UV_PYTHON_INSTALL_DIR=/tmp/build_cache/py_runtime uv python install $LATEST_STABLE_PY +# --compile-bytecode precompiles the stdlib to .pyc so jobs don't recompile it on every run +# under the read-only nsjail runtime mount (uv >= 0.9.25). The copy below MUST preserve +# timestamps or Python's mtime-based .pyc invalidation discards these compiled files. +RUN UV_CACHE_DIR=/tmp/build_cache/uv UV_PYTHON_INSTALL_DIR=/tmp/build_cache/py_runtime uv python install 3.11 --compile-bytecode +RUN UV_CACHE_DIR=/tmp/build_cache/uv UV_PYTHON_INSTALL_DIR=/tmp/build_cache/py_runtime uv python install $LATEST_STABLE_PY --compile-bytecode RUN curl -sL https://deb.nodesource.com/setup_20.x | bash - @@ -259,7 +262,7 @@ RUN export GOCACHE=/tmp/build_cache/go && \ # chmod a+rw adds read+write WITHOUT removing execute bits (755->777, 644->666) # Note: uv python install only creates py_runtime, not uv cache - we create uv/go dirs for runtime RUN mkdir -p /tmp/windmill/cache && \ - cp -r /tmp/build_cache/* /tmp/windmill/cache/ && \ + cp -r --preserve=timestamps /tmp/build_cache/* /tmp/windmill/cache/ && \ chmod -R a+rw /tmp/windmill/cache && \ rm -rf /tmp/build_cache && \ mkdir -p -m 777 /tmp/windmill/cache/uv /tmp/windmill/cache/go /tmp/windmill/cache/rustup /tmp/windmill/cache/cargo diff --git a/backend/windmill-worker/nsjail/download.py.config.proto b/backend/windmill-worker/nsjail/download.py.config.proto index 217fe5763a..18957bb4a5 100644 --- a/backend/windmill-worker/nsjail/download.py.config.proto +++ b/backend/windmill-worker/nsjail/download.py.config.proto @@ -5,7 +5,14 @@ hostname: "python" log_level: ERROR time_limit: 900 -rlimit_as: 2048 +# uv's --compile-bytecode spawns a bytecode-compile thread pool sized to the +# host's CPU count. Each thread reserves virtual address space for its stack, so +# on high-core machines the aggregate overruns a low rlimit_as and installs fail +# intermittently with "OS can't spawn worker thread: Resource temporarily +# unavailable (os error 11)" / "memory allocation failed". A low cap (was 2048) +# is the address-space companion to the fd exhaustion fixed below; raised well +# above the run sandbox's 4096 to give the compile pool headroom on large nodes. +rlimit_as: 8192 rlimit_cpu: 1000 rlimit_fsize: 1024 # uv's --compile-bytecode spawns a Python interpreter that compiles .py files diff --git a/backend/windmill-worker/src/python_versions.rs b/backend/windmill-worker/src/python_versions.rs index 5d06d3e673..0ae6b3462a 100644 --- a/backend/windmill-worker/src/python_versions.rs +++ b/backend/windmill-worker/src/python_versions.rs @@ -537,6 +537,11 @@ impl PyV { &v, "--python-preference=only-managed", "--no-bin", + // Compile the runtime's stdlib to bytecode at install time. The + // runtime is mounted read-only into the job nsjail, so without + // precompiled .pyc Python would recompile ~stdlib from source on + // every job (and can never persist it). Requires uv >= 0.9.25. + "--compile-bytecode", ]) // TODO: Do we need these? .envs([ diff --git a/docker/DockerfileSlim b/docker/DockerfileSlim index b192a46c34..91e64d9fb4 100644 --- a/docker/DockerfileSlim +++ b/docker/DockerfileSlim @@ -54,14 +54,17 @@ RUN curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmo ENV TZ=Etc/UTC # Install UV -RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.9.24/uv-installer.sh | sh && mv /root/.local/bin/uv /usr/local/bin/uv +RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.9.25/uv-installer.sh | sh && mv /root/.local/bin/uv /usr/local/bin/uv # Preinstall python runtime to temp location (will copy with world-writable perms later) -RUN UV_PYTHON_INSTALL_DIR=/tmp/build_cache/py_runtime uv python install $LATEST_STABLE_PY +# --compile-bytecode precompiles the stdlib to .pyc so jobs don't recompile it on every run +# under the read-only nsjail runtime mount (uv >= 0.9.25). The copy below MUST preserve +# timestamps or Python's mtime-based .pyc invalidation discards these compiled files. +RUN UV_PYTHON_INSTALL_DIR=/tmp/build_cache/py_runtime uv python install $LATEST_STABLE_PY --compile-bytecode # Copy to final location with world-writable permissions for arbitrary UID support RUN mkdir -p /tmp/windmill/cache && \ - cp -r /tmp/build_cache/* /tmp/windmill/cache/ && \ + cp -r --preserve=timestamps /tmp/build_cache/* /tmp/windmill/cache/ && \ chmod -R a+rw /tmp/windmill/cache && \ rm -rf /tmp/build_cache && \ mkdir -p -m 777 /tmp/windmill/cache/uv diff --git a/docker/DockerfileSlimEe b/docker/DockerfileSlimEe index 24d93586f8..15366bfe06 100644 --- a/docker/DockerfileSlimEe +++ b/docker/DockerfileSlimEe @@ -54,14 +54,17 @@ RUN curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmo ENV TZ=Etc/UTC # Install UV -RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.9.24/uv-installer.sh | sh && mv /root/.local/bin/uv /usr/local/bin/uv +RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.9.25/uv-installer.sh | sh && mv /root/.local/bin/uv /usr/local/bin/uv # Preinstall python runtime to temp location (will copy with world-writable perms later) -RUN UV_PYTHON_INSTALL_DIR=/tmp/build_cache/py_runtime uv python install $LATEST_STABLE_PY +# --compile-bytecode precompiles the stdlib to .pyc so jobs don't recompile it on every run +# under the read-only nsjail runtime mount (uv >= 0.9.25). The copy below MUST preserve +# timestamps or Python's mtime-based .pyc invalidation discards these compiled files. +RUN UV_PYTHON_INSTALL_DIR=/tmp/build_cache/py_runtime uv python install $LATEST_STABLE_PY --compile-bytecode # Copy to final location with world-writable permissions for arbitrary UID support RUN mkdir -p /tmp/windmill/cache && \ - cp -r /tmp/build_cache/* /tmp/windmill/cache/ && \ + cp -r --preserve=timestamps /tmp/build_cache/* /tmp/windmill/cache/ && \ chmod -R a+rw /tmp/windmill/cache && \ rm -rf /tmp/build_cache && \ mkdir -p -m 777 /tmp/windmill/cache/uv From 8053266f88bd4c94fc86278412df5a0beeed5e77 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 3 Jun 2026 11:00:43 +0200 Subject: [PATCH 23/61] fix(mcp): resolve MCP resource token via caller RLS + SSRF-guard url (#9428) * fix(mcp): resolve MCP resource token via caller RLS + SSRF-guard url Co-Authored-By: Claude Opus 4.8 (1M context) * fix(mcp): clone user_db for oauth2 refresh and drop advisory ids from comments Co-Authored-By: Claude Opus 4.8 (1M context) * fix(mcp): disable redirects on MCP client to prevent SSRF bypass Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Claude Opus 4.8 (1M context) --- backend/tests/fixtures/mcp_token_exfil.sql | 29 ++++++ backend/tests/mcp_token_exfil.rs | 111 +++++++++++++++++++++ backend/windmill-api/src/mcp_tools.rs | 24 ++++- backend/windmill-mcp/Cargo.toml | 3 + backend/windmill-mcp/src/client/mod.rs | 64 ++++++++++-- backend/windmill-worker/src/ai/utils.rs | 52 +++++++--- backend/windmill-worker/src/ai_executor.rs | 2 +- 7 files changed, 255 insertions(+), 30 deletions(-) create mode 100644 backend/tests/fixtures/mcp_token_exfil.sql create mode 100644 backend/tests/mcp_token_exfil.rs diff --git a/backend/tests/fixtures/mcp_token_exfil.sql b/backend/tests/fixtures/mcp_token_exfil.sql new file mode 100644 index 0000000000..edf1113137 --- /dev/null +++ b/backend/tests/fixtures/mcp_token_exfil.sql @@ -0,0 +1,29 @@ +-- Fixture for the MCP token-exfiltration regression test. +-- +-- Models a malicious developer (test-user-3, a plain workspace member) who: +-- - owns an MCP resource they are allowed to read, and +-- - points that resource's `token` field at a secret variable living in a +-- folder they have NO access to (`f/locked`, only test-user/admin owns it). +-- +-- The secret variable `f/locked/secret_token` itself is inserted by the test in +-- Rust (so it is encrypted with the real workspace key); this fixture only sets +-- up the locked folder, the resource, and their permissions. + +-- Folder the developer cannot read (empty extra_perms, owned by admin only). +INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms, created_by) +VALUES ('test-workspace', 'locked', 'Locked Folder', '{"u/test-user"}', '{}', 'test-user'); + +-- MCP resource owned by the developer (so RLS lets them read the resource), +-- whose token references the locked secret. The URL is a non-resolvable public +-- host so that, for an authorized caller, resolution succeeds but the later +-- connection/SSRF step fails deterministically without network access. +INSERT INTO resource (workspace_id, path, value, description, resource_type, extra_perms, created_by) +VALUES ( + 'test-workspace', + 'u/test-user-3/evil_mcp', + '{"name": "evil", "url": "https://mcp.invalid.windmill.test", "token": "$var:f/locked/secret_token"}', + 'MCP resource whose token points at a locked secret', + 'mcp', + '{}', + 'test-user-3' +); diff --git a/backend/tests/mcp_token_exfil.rs b/backend/tests/mcp_token_exfil.rs new file mode 100644 index 0000000000..ce278f3c53 --- /dev/null +++ b/backend/tests/mcp_token_exfil.rs @@ -0,0 +1,111 @@ +//! Regression test for the MCP token-exfiltration vulnerability. +//! +//! `GET /api/w/{w}/resources/mcp_tools/{path}` builds an MCP client from a +//! resource whose `token` field is a `$var:` reference. Before the fix the token +//! was resolved with `get_secret_value_as_admin` on the bare DB pool — no RLS, +//! no audit — so any workspace member who could read an MCP *resource* could +//! point its token at *any* secret variable in the workspace (e.g. one in an +//! admin-only folder) and have it decrypted and shipped as a bearer token. +//! +//! The fix resolves the token through the caller's permissioned path +//! (`get_value_internal` over the authed `user_db`), so the variable RLS — the +//! same gate as `variables/get_value` — applies and the secret read is audited. +//! +//! This test pins, against the `mcp_token_exfil` fixture: +//! - a plain developer (test-user-3) who can read the MCP resource but has no +//! access to the locked secret is DENIED (401) at token resolution, before +//! any connection is attempted, and the secret never leaks; +//! - an admin (test-user) clears the variable-RLS gate, the token resolves, +//! and the request only fails later at the connect/SSRF step — proving the +//! legitimate path still resolves the token (no over-blocking). +//! +//! SSRF rejection of an author-controlled URL is covered by the unit test in +//! `windmill-mcp` (`from_resource_rejects_ssrf_url`). +#![cfg(feature = "mcp")] + +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +const SECRET_VALUE: &str = "S3CRET-MCP-TOKEN-VALUE"; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +async fn get(base: &str, path: &str, token: &str) -> (reqwest::StatusCode, String) { + let resp = client() + .get(format!("{base}/{path}")) + .header("Authorization", format!("Bearer {token}")) + .send() + .await + .expect("request"); + let status = resp.status(); + let body = resp.text().await.expect("body"); + (status, body) +} + +#[sqlx::test(fixtures("base", "mcp_token_exfil"))] +async fn test_mcp_token_not_exfiltrated(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + // Insert the locked secret variable with a real, workspace-key-encrypted + // value so an authorized read genuinely decrypts it. + let mc = windmill_common::variables::build_crypt(&db, "test-workspace").await?; + let encrypted = windmill_common::variables::encrypt(&mc, SECRET_VALUE); + // Runtime-checked query (not the `query!` macro) so no offline `.sqlx` cache + // entry is needed for this test-only insert. + sqlx::query( + "INSERT INTO variable (workspace_id, path, value, is_secret, description, extra_perms) + VALUES ('test-workspace', 'f/locked/secret_token', $1, true, 'Locked secret', '{}')", + ) + .bind(&encrypted) + .execute(&db) + .await?; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/resources/mcp_tools"); + let path = "u/test-user-3/evil_mcp"; + + // ---- CORE REGRESSION: the developer can read the resource but must NOT be + // able to resolve the locked secret. They are denied (401) at the + // variable-RLS gate, before any MCP connection is attempted, and the + // secret never appears in the response. + let (status, body) = get(&base, path, "SECRET_TOKEN_3").await; + assert_eq!( + status, + reqwest::StatusCode::UNAUTHORIZED, + "developer must be denied resolving a secret they can't read (got {status}): {body}" + ); + assert!( + !body.contains(SECRET_VALUE), + "the locked secret must never leak to the developer: {body}" + ); + assert!( + body.contains("don't have access"), + "denial should come from the variable-RLS gate, not a connection error: {body}" + ); + // Pre-fix, the token was decrypted as admin and the handler proceeded to the + // connection step; that path must no longer be reached for the developer. + assert!( + !body.contains("Failed to connect to MCP server"), + "developer must be blocked before the connection step (would mean the token was resolved): {body}" + ); + + // ---- NO OVER-BLOCKING: an admin clears the variable-RLS gate, so the token + // resolves and the request only fails later at the connect/SSRF step. + // A different failure mode (not 401, reaches the connection) proves the + // legitimate read still works. + let (status, body) = get(&base, path, "SECRET_TOKEN").await; + assert_ne!( + status, + reqwest::StatusCode::UNAUTHORIZED, + "admin must clear the variable-RLS gate (got {status}): {body}" + ); + assert!( + body.contains("Failed to connect to MCP server"), + "admin should resolve the token and only fail at the connect/SSRF step: {body}" + ); + + Ok(()) +} diff --git a/backend/windmill-api/src/mcp_tools.rs b/backend/windmill-api/src/mcp_tools.rs index 10397f0dee..bba945ac54 100644 --- a/backend/windmill-api/src/mcp_tools.rs +++ b/backend/windmill-api/src/mcp_tools.rs @@ -5,11 +5,11 @@ use axum::{ use serde_json::value::RawValue; use windmill_api_auth::{check_scopes, ApiAuthed}; use windmill_common::{ - db::{UserDB, DB}, + db::{DbWithOptAuthed, UserDB, DB}, error::{Error, JsonResult, Result}, utils::{not_found_if_none, StripPath}, }; -use windmill_store::resources::explain_resource_perm_error; +use windmill_store::{resources::explain_resource_perm_error, variables::get_value_internal}; pub(crate) async fn get_mcp_tools( authed: ApiAuthed, @@ -65,7 +65,7 @@ pub(crate) async fn get_mcp_tools( if let Some(info) = token_info { if let (Some(account_id), Some(true)) = (info.account_id, info.is_expired) { - let refresh_tx = user_db.begin(&authed).await?; + let refresh_tx = user_db.clone().begin(&authed).await?; if let Err(e) = crate::oauth2_oss::_refresh_token( refresh_tx, token_var_path, @@ -85,7 +85,23 @@ pub(crate) async fn get_mcp_tools( } } - let client = windmill_mcp::McpClient::from_resource(mcp_resource, &db, &w_id) + // Resolve the token through the caller's permissioned (RLS + audit) path so + // a developer cannot exfiltrate a secret they are not allowed to read by + // pointing an MCP resource's token at it. + let token = if let Some(token_path) = &mcp_resource.token { + let token_var_path = token_path.trim_start_matches("$var:"); + if token_var_path.trim().is_empty() { + None + } else { + let db_authed = + DbWithOptAuthed::from_authed(&authed, db.clone(), Some(user_db.clone())); + Some(get_value_internal(&db_authed, &w_id, token_var_path, false).await?) + } + } else { + None + }; + + let client = windmill_mcp::McpClient::from_resource(mcp_resource, token) .await .map_err(|e| Error::ExecutionErr(format!("Failed to connect to MCP server: {}", e)))?; diff --git a/backend/windmill-mcp/Cargo.toml b/backend/windmill-mcp/Cargo.toml index 3968ea0836..36e0d03d18 100644 --- a/backend/windmill-mcp/Cargo.toml +++ b/backend/windmill-mcp/Cargo.toml @@ -29,3 +29,6 @@ http = { workspace = true, optional = true } tokio-util = { workspace = true, features = ["rt"], optional = true } tokio = { workspace = true, optional = true } futures.workspace = true + +[dev-dependencies] +tokio = { workspace = true, features = ["macros", "rt"] } diff --git a/backend/windmill-mcp/src/client/mod.rs b/backend/windmill-mcp/src/client/mod.rs index bc6d2c24ac..1a555c141f 100644 --- a/backend/windmill-mcp/src/client/mod.rs +++ b/backend/windmill-mcp/src/client/mod.rs @@ -22,8 +22,6 @@ use rmcp::{ }; use serde_json::{json, Value}; use std::str::FromStr; -use windmill_common::variables::get_secret_value_as_admin; -use windmill_common::DB; /// MCP client for communicating with external MCP servers pub struct McpClient { @@ -34,18 +32,29 @@ pub struct McpClient { } impl McpClient { - /// Create a new MCP client from a resource configuration - pub async fn from_resource(resource: McpResource, db: &DB, w_id: &str) -> Result { + /// Create a new MCP client from a resource configuration. + /// + /// `token`, when present, is the already-resolved bearer token sent as an + /// `Authorization` header. It MUST be resolved by the caller through the + /// permissioned (RLS + audit) variable path — `from_resource` never reads + /// secrets itself, so a caller cannot trick it into decrypting a variable + /// they are not allowed to read. + pub async fn from_resource(resource: McpResource, token: Option) -> Result { + // The resource URL is author-controlled and we send a (potentially + // secret) bearer token to it, so it must be validated against SSRF + // before we connect (e.g. cloud metadata endpoints, internal services). + windmill_common::ssrf::validate_url_for_ssrf(&resource.url) + .await + .map_err(|e| anyhow::anyhow!("MCP server URL is not allowed: {}", e))?; + // Build custom reqwest client with headers if provided let mut headers = HeaderMap::new(); - if let Some(token_path) = &resource.token { - if !token_path.trim().is_empty() { - let value = - get_secret_value_as_admin(db, w_id, token_path.trim_start_matches("$var:")) - .await?; + if let Some(token) = token { + let token = token.trim(); + if !token.is_empty() { headers.insert( HeaderName::from_static("authorization"), - HeaderValue::from_str(format!("Bearer {}", value).as_str())?, + HeaderValue::from_str(format!("Bearer {}", token).as_str())?, ); } } @@ -64,6 +73,12 @@ impl McpClient { let reqwest_client = reqwest::Client::builder() .default_headers(headers) + // Don't follow redirects: the SSRF check above only validates the + // initial (author-controlled) URL, so following a redirect could + // still reach a private/internal address with the bearer token + // attached. The MCP streamable-HTTP endpoint is a direct endpoint + // and does not legitimately rely on redirects. + .redirect(reqwest::redirect::Policy::none()) .build() .context("Failed to build HTTP client")?; @@ -210,3 +225,32 @@ impl McpClient { } } } + +#[cfg(test)] +mod tests { + use super::*; + + /// Regression test: `from_resource` must refuse to connect to a URL that + /// targets a private/internal address (here the AWS + /// instance-metadata endpoint), so a resource author cannot use the MCP + /// client as an SSRF primitive against internal services. The guard runs + /// before any connection attempt, so this fails fast without network access. + #[tokio::test] + async fn from_resource_rejects_ssrf_url() { + let resource = McpResource { + name: "evil".to_string(), + url: "http://169.254.169.254".to_string(), + token: None, + headers: None, + }; + + let msg = match McpClient::from_resource(resource, None).await { + Ok(_) => panic!("a link-local metadata URL must be rejected before connecting"), + Err(e) => e.to_string(), + }; + assert!( + msg.contains("not allowed") && msg.contains("private"), + "error should explain the URL was rejected as private/internal, got: {msg}" + ); + } +} diff --git a/backend/windmill-worker/src/ai/utils.rs b/backend/windmill-worker/src/ai/utils.rs index 74e75ef0a5..353bab17a0 100644 --- a/backend/windmill-worker/src/ai/utils.rs +++ b/backend/windmill-worker/src/ai/utils.rs @@ -7,6 +7,8 @@ use std::{ }; use uuid::Uuid; use windmill_ai::types::*; +#[cfg(feature = "mcp")] +use windmill_common::client::AuthedClient; use windmill_common::flows::FlowModuleValue; use windmill_common::{ db::DB, @@ -546,7 +548,7 @@ pub async fn load_mcp_tools( db: &DB, workspace_id: &str, mcp_configs: Vec, - auth_token: &str, + client: &AuthedClient, ) -> Result<(HashMap>, Vec), Error> { let mut all_mcp_tools = Vec::new(); let mut mcp_clients = HashMap::new(); @@ -573,27 +575,47 @@ pub async fn load_mcp_tools( let resource_name = mcp_resource.name.clone(); - // Check if token needs refresh before creating MCP client - if let Some(ref token_path) = mcp_resource.token { + // Resolve the token through the job's permissioned (RLS + audit) path so + // the AI agent cannot exfiltrate a secret its identity is not allowed to + // read by pointing an MCP resource's token at it. + let token = if let Some(ref token_path) = mcp_resource.token { let token_var_path = token_path.trim_start_matches("$var:"); - if let Err(e) = - refresh_token_if_expired(db, workspace_id, token_var_path, auth_token).await - { - tracing::warn!( - "Failed to refresh token for MCP resource {}: {}. Proceeding with possibly expired token.", - resource_name, e - ); + if token_var_path.trim().is_empty() { + None + } else { + // Refresh first (best-effort) so the value we read is current. + if let Err(e) = + refresh_token_if_expired(db, workspace_id, token_var_path, &client.token).await + { + tracing::warn!( + "Failed to refresh token for MCP resource {}: {}. Proceeding with possibly expired token.", + resource_name, e + ); + } + Some( + client + .get_variable_value(token_var_path) + .await + .map_err(|e| { + Error::internal_err(format!( + "Failed to resolve token variable {} for MCP resource {}: {}", + token_var_path, resource_name, e + )) + })?, + ) } - } + } else { + None + }; // Create new MCP client for this execution tracing::debug!("Creating fresh MCP client for {}", resource_name); - let client = McpClient::from_resource(mcp_resource, db, workspace_id) + let mcp_conn = McpClient::from_resource(mcp_resource, token) .await .context("Failed to create MCP client")?; // Get raw MCP tools from client - let raw_mcp_tools = client.available_tools(); + let raw_mcp_tools = mcp_conn.available_tools(); // Convert to Windmill Tool format let converted_tools = @@ -616,7 +638,7 @@ pub async fn load_mcp_tools( all_mcp_tools.extend(filtered_tools); // Store client for later use and cleanup - let mcp_client = Arc::new(client); + let mcp_client = Arc::new(mcp_conn); mcp_clients.insert(resource_name, mcp_client); } @@ -663,7 +685,7 @@ pub async fn load_mcp_tools( _db: &DB, _workspace_id: &str, _mcp_configs: Vec, - _auth_token: &str, + _client: &windmill_common::client::AuthedClient, ) -> Result<(HashMap>, Vec), Error> { Ok((HashMap::new(), Vec::new())) } diff --git a/backend/windmill-worker/src/ai_executor.rs b/backend/windmill-worker/src/ai_executor.rs index b2478e3be1..2591cc6495 100644 --- a/backend/windmill-worker/src/ai_executor.rs +++ b/backend/windmill-worker/src/ai_executor.rs @@ -432,7 +432,7 @@ pub async fn handle_ai_agent_job( let mcp_clients = if !mcp_configs.is_empty() { let (clients, mcp_tools) = - load_mcp_tools(db, &job.workspace_id, mcp_configs, &client.token).await?; + load_mcp_tools(db, &job.workspace_id, mcp_configs, client).await?; tools.extend(mcp_tools); clients } else { From 11d1ad9a872d2ec2f14cde35708c84a0c7bdc172 Mon Sep 17 00:00:00 2001 From: centdix <40307056+centdix@users.noreply.github.com> Date: Wed, 3 Jun 2026 11:29:21 +0200 Subject: [PATCH 24/61] fix: omit temperature for gpt-5+ and o-series models on all providers (#9422) Co-authored-by: Claude Opus 4.8 (1M context) --- .../src/lib/components/copilot/lib.test.ts | 51 +++++++++++++++++++ frontend/src/lib/components/copilot/lib.ts | 4 +- .../src/lib/components/copilot/modelConfig.ts | 15 +++++- 3 files changed, 67 insertions(+), 3 deletions(-) diff --git a/frontend/src/lib/components/copilot/lib.test.ts b/frontend/src/lib/components/copilot/lib.test.ts index 369987d53f..541bc60b3d 100644 --- a/frontend/src/lib/components/copilot/lib.test.ts +++ b/frontend/src/lib/components/copilot/lib.test.ts @@ -28,6 +28,13 @@ describe('modelConfig', () => { expect(modelDisallowsSamplingParams('anthropic/claude-opus-4-7')).toBe(true) }) + it('flags Opus 4.8 model IDs via includes matching', () => { + expect(modelDisallowsSamplingParams('claude-opus-4-8')).toBe(true) + expect(modelDisallowsSamplingParams('claude-opus-4-8@20260416')).toBe(true) + expect(modelDisallowsSamplingParams('claude-opus-4-8/thinking')).toBe(true) + expect(modelDisallowsSamplingParams('anthropic/claude-opus-4-8')).toBe(true) + }) + it('omits deterministic temperature for Anthropic Opus 4.7 chat requests', () => { expect( getDefaultChatTemperature({ provider: 'anthropic', model: 'claude-opus-4-7' }) @@ -43,6 +50,50 @@ describe('modelConfig', () => { it('keeps deterministic temperature for older Anthropic models', () => { expect(getDefaultChatTemperature({ provider: 'anthropic', model: 'claude-sonnet-4-6' })).toBe(0) }) + + it('flags gpt-5+ and o-series reasoning models via prefix matching', () => { + expect(modelDisallowsSamplingParams('gpt-5')).toBe(true) + expect(modelDisallowsSamplingParams('gpt-5.5')).toBe(true) + expect(modelDisallowsSamplingParams('gpt-5-mini')).toBe(true) + expect(modelDisallowsSamplingParams('o1')).toBe(true) + expect(modelDisallowsSamplingParams('o3')).toBe(true) + expect(modelDisallowsSamplingParams('o4-mini')).toBe(true) + // provider-prefixed identifiers (e.g. OpenRouter) match on the bare model id + expect(modelDisallowsSamplingParams('openai/gpt-5')).toBe(true) + expect(modelDisallowsSamplingParams('openai/o3')).toBe(true) + }) + + it('keeps sampling params for non-reasoning models that merely share a prefix', () => { + // gpt-4o starts with "gpt-" but not "gpt-5"; the "o" is mid-string, not a prefix + expect(modelDisallowsSamplingParams('gpt-4o')).toBe(false) + expect(modelDisallowsSamplingParams('gpt-4o-mini')).toBe(false) + // the provider prefix "openai/" must not be mistaken for an o-series model + expect(modelDisallowsSamplingParams('openai/gpt-4o')).toBe(false) + // the o-series match requires a digit after "o", so non-OpenAI ids that + // start with "o" (Mistral open-* family, OpenRouter optimus-*/openchat-*) + // keep their deterministic temperature + expect(modelDisallowsSamplingParams('open-mistral-7b')).toBe(false) + expect(modelDisallowsSamplingParams('open-mixtral-8x7b')).toBe(false) + expect(modelDisallowsSamplingParams('open-mistral-nemo-2407')).toBe(false) + expect(modelDisallowsSamplingParams('optimus-alpha')).toBe(false) + expect(modelDisallowsSamplingParams('openchat/openchat-7b')).toBe(false) + }) + + it('keeps deterministic temperature for Mistral open-* models', () => { + expect(getDefaultChatTemperature({ provider: 'mistral', model: 'open-mixtral-8x7b' })).toBe(0) + }) + + it('omits deterministic temperature for gpt-5.5 routed through the customai gateway', () => { + expect(getDefaultChatTemperature({ provider: 'customai', model: 'gpt-5.5' })).toBeUndefined() + }) + + it('omits deterministic temperature for o-series models on the customai gateway', () => { + expect(getDefaultChatTemperature({ provider: 'customai', model: 'o3' })).toBeUndefined() + }) + + it('keeps deterministic temperature for gpt-4o on the customai gateway', () => { + expect(getDefaultChatTemperature({ provider: 'customai', model: 'gpt-4o' })).toBe(0) + }) }) describe('fim autocomplete', () => { diff --git a/frontend/src/lib/components/copilot/lib.ts b/frontend/src/lib/components/copilot/lib.ts index 00d2334acc..b87fb76ca6 100644 --- a/frontend/src/lib/components/copilot/lib.ts +++ b/frontend/src/lib/components/copilot/lib.ts @@ -14,7 +14,7 @@ import Anthropic from '@anthropic-ai/sdk' import { get, type Writable } from 'svelte/store' import { OpenAPI, ResourceService, type Script } from '../../gen' import { EDIT_CONFIG, FIX_CONFIG, GEN_CONFIG } from './prompts' -import { getDefaultChatTemperature } from './modelConfig' +import { getDefaultChatTemperature, modelDisallowsSamplingParams } from './modelConfig' import { formatResourceTypes } from './utils' import { processToolCall, type Tool, type ToolCallbacks } from './chat/shared' import { @@ -317,7 +317,7 @@ function getModelSpecificConfig( const defaultTemperature = getDefaultChatTemperature(modelProvider) if ( (modelProvider.provider === 'openai' || modelProvider.provider === 'azure_openai') && - (modelProvider.model.startsWith('o') || modelProvider.model.startsWith('gpt-5')) + modelDisallowsSamplingParams(modelProvider.model) ) { return { model: modelProvider.model, diff --git a/frontend/src/lib/components/copilot/modelConfig.ts b/frontend/src/lib/components/copilot/modelConfig.ts index 40361024c2..e80b7ce7a5 100644 --- a/frontend/src/lib/components/copilot/modelConfig.ts +++ b/frontend/src/lib/components/copilot/modelConfig.ts @@ -2,7 +2,20 @@ import type { AIProviderModel } from '$lib/gen' export function modelDisallowsSamplingParams(model: string) { const normalizedModel = model.toLowerCase() - return normalizedModel.includes('claude-opus-4-7') + // Strip any provider prefix (e.g. OpenRouter's "openai/o3") so the + // reasoning-model check matches the bare model id rather than the prefix. + const baseModel = normalizedModel.split('/').pop() ?? normalizedModel + // gpt-5+ and o-series reasoning models reject sampling params such as + // temperature (only the default value is supported), regardless of which + // provider/gateway routes the request — so this must stay provider-agnostic. + // The o-series match requires a digit after the "o" (o1/o3/o4-mini) so it + // does not catch unrelated ids like Mistral's "open-mistral-*" or "optimus-*". + return ( + normalizedModel.includes('claude-opus-4-7') || + normalizedModel.includes('claude-opus-4-8') || + baseModel.startsWith('gpt-5') || + /^o\d/.test(baseModel) + ) } export function getDefaultChatTemperature(modelProvider: AIProviderModel): number | undefined { From 47c96204deadb82909aa8c7bcc9e254df30afc08 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 3 Jun 2026 12:08:16 +0200 Subject: [PATCH 25/61] chore(main): release 1.715.0 (#9421) * chore(main): release 1.715.0 * Apply automatic changes --------- Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com> --- CHANGELOG.md | 16 ++ backend/Cargo.lock | 180 ++++++++++-------- backend/Cargo.toml | 4 +- .../parsers/windmill-parser-wasm/Cargo.lock | 48 ++--- .../parsers/windmill-parser-wasm/Cargo.toml | 2 +- backend/windmill-api/openapi.yaml | 2 +- benchmarks/lib.ts | 2 +- cli/src/main.ts | 2 +- frontend/package-lock.json | 4 +- frontend/package.json | 2 +- lsp/Pipfile | 2 +- openflow.openapi.yaml | 2 +- .../WindmillClient/WindmillClient.psd1 | 2 +- python-client/wmill/pyproject.toml | 2 +- typescript-client/jsr.json | 2 +- typescript-client/package.json | 2 +- version.txt | 2 +- 17 files changed, 155 insertions(+), 121 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 719af698d7..f58de02f31 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,21 @@ # Changelog +## [1.715.0](https://github.com/windmill-labs/windmill/compare/v1.714.1...v1.715.0) (2026-06-03) + + +### Features + +* **frontend:** add rebuild dependency map button to workspace settings ([#9424](https://github.com/windmill-labs/windmill/issues/9424)) ([3b2e748](https://github.com/windmill-labs/windmill/commit/3b2e748daf0a8ec4447c30423068df803f3f9ca2)) + + +### Bug Fixes + +* **auth:** filter script/flow listings by token scope (GHSA-2ppx-66jv-wpw5) ([#9426](https://github.com/windmill-labs/windmill/issues/9426)) ([7edf3f0](https://github.com/windmill-labs/windmill/commit/7edf3f02122e20fde1e95e0252e7bda641075326)) +* **backend:** authorize single-job read endpoints by job/flow visibility ([#9416](https://github.com/windmill-labs/windmill/issues/9416)) ([89a7a37](https://github.com/windmill-labs/windmill/commit/89a7a377764086911db18252f2478f42f0e1e3ea)) +* **mcp:** resolve MCP resource token via caller RLS + SSRF-guard url ([#9428](https://github.com/windmill-labs/windmill/issues/9428)) ([8053266](https://github.com/windmill-labs/windmill/commit/8053266f88bd4c94fc86278412df5a0beeed5e77)) +* **nsjail:** precompile python stdlib + raise download rlimit_as ([#9429](https://github.com/windmill-labs/windmill/issues/9429)) ([7031744](https://github.com/windmill-labs/windmill/commit/7031744a199f0bf8b8e35043afa959977e5ecdbd)) +* omit temperature for gpt-5+ and o-series models on all providers ([#9422](https://github.com/windmill-labs/windmill/issues/9422)) ([11d1ad9](https://github.com/windmill-labs/windmill/commit/11d1ad9a872d2ec2f14cde35708c84a0c7bdc172)) + ## [1.714.1](https://github.com/windmill-labs/windmill/compare/v1.714.0...v1.714.1) (2026-06-02) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index c9e09a4963..567af4bbcc 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -6372,6 +6372,16 @@ version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4933f3f57a8e9d9da04db23fb153356ecaf00cbd14aee46279c33dc80925c37" +[[package]] +name = "kstat-rs" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27964e4632377753acb0898ce6f28770d50cbca1339200ae63d700cff97b5c2b" +dependencies = [ + "libc", + "thiserror 1.0.69", +] + [[package]] name = "kube" version = "1.1.0" @@ -6817,6 +6827,12 @@ dependencies = [ "libc", ] +[[package]] +name = "mach2" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dae608c151f68243f2b000364e1f7b186d9c29845f7d2d85bd31b9ad77ad552b" + [[package]] name = "macro_rules_attribute" version = "0.2.2" @@ -7487,7 +7503,7 @@ dependencies = [ "libc", "libproc", "log", - "mach2", + "mach2 0.4.3", "nix 0.29.0", "ntapi", "procfs", @@ -11805,13 +11821,15 @@ dependencies = [ [[package]] name = "systemstat" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6e89b75de097d0c52a1dc2114e19439d55f0e2e42d32168c6df44f139dfb66f" +checksum = "a583abe520746270ffdbdaf0e3039a806f29be9d7034d66466a4839a01de0610" dependencies = [ "bytesize", + "kstat-rs", "lazy_static", "libc", + "mach2 0.6.0", "nom", "time", "winapi", @@ -13764,7 +13782,7 @@ dependencies = [ [[package]] name = "windmill" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "async-nats", @@ -13845,7 +13863,7 @@ dependencies = [ [[package]] name = "windmill-ai" -version = "1.714.1" +version = "1.715.0" dependencies = [ "async-stream", "async-trait", @@ -13878,7 +13896,7 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.714.1" +version = "1.715.0" dependencies = [ "axum 0.8.9", "chrono", @@ -13891,7 +13909,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "argon2", @@ -14029,7 +14047,7 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.714.1" +version = "1.715.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14052,7 +14070,7 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.714.1" +version = "1.715.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14065,7 +14083,7 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14091,7 +14109,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.714.1" +version = "1.715.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -14101,7 +14119,7 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.714.1" +version = "1.715.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14118,7 +14136,7 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.714.1" +version = "1.715.0" dependencies = [ "axum 0.8.9", "base64 0.22.1", @@ -14140,7 +14158,7 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14163,7 +14181,7 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.714.1" +version = "1.715.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14179,7 +14197,7 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.714.1" +version = "1.715.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14200,7 +14218,7 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.714.1" +version = "1.715.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14221,7 +14239,7 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.714.1" +version = "1.715.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14235,7 +14253,7 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "async-nats", @@ -14267,7 +14285,7 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14292,7 +14310,7 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.714.1" +version = "1.715.0" dependencies = [ "axum 0.8.9", "flate2", @@ -14310,7 +14328,7 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14332,7 +14350,7 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.714.1" +version = "1.715.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14352,7 +14370,7 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.714.1" +version = "1.715.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14382,7 +14400,7 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14410,7 +14428,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.714.1" +version = "1.715.0" dependencies = [ "lazy_static", "serde", @@ -14422,7 +14440,7 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.714.1" +version = "1.715.0" dependencies = [ "argon2", "axum 0.8.9", @@ -14447,7 +14465,7 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.714.1" +version = "1.715.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14461,7 +14479,7 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.714.1" +version = "1.715.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14494,7 +14512,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.714.1" +version = "1.715.0" dependencies = [ "chrono", "lazy_static", @@ -14508,7 +14526,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14527,7 +14545,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.714.1" +version = "1.715.0" dependencies = [ "aes-gcm", "aho-corasick", @@ -14628,7 +14646,7 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.714.1" +version = "1.715.0" dependencies = [ "chrono", "itertools 0.14.0", @@ -14647,7 +14665,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.714.1" +version = "1.715.0" dependencies = [ "regex", "serde", @@ -14662,7 +14680,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "astral-tokio-tar", @@ -14686,7 +14704,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "futures", @@ -14703,7 +14721,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.714.1" +version = "1.715.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -14719,7 +14737,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "async-trait", @@ -14740,7 +14758,7 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "async-trait", @@ -14771,7 +14789,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "arc-swap", @@ -14796,7 +14814,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "async-stream", @@ -14830,7 +14848,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "futures", @@ -14848,7 +14866,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.714.1" +version = "1.715.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -14857,7 +14875,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "lazy_static", @@ -14869,7 +14887,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "serde_json", @@ -14881,7 +14899,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "gosyn", @@ -14893,7 +14911,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "lazy_static", @@ -14905,7 +14923,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "serde_json", @@ -14917,7 +14935,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "nu-parser", @@ -14928,7 +14946,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -14939,7 +14957,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -14951,7 +14969,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "rustpython-ast", @@ -14962,7 +14980,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "async-recursion", @@ -14984,7 +15002,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "serde_json", @@ -14996,7 +15014,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "lazy_static", @@ -15010,7 +15028,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -15027,7 +15045,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "lazy_static", @@ -15040,7 +15058,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "serde", @@ -15052,7 +15070,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "lazy_static", @@ -15070,7 +15088,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -15086,7 +15104,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "rustpython-ast", @@ -15102,7 +15120,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "serde", @@ -15113,7 +15131,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "async-recursion", @@ -15151,7 +15169,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "const_format", @@ -15189,7 +15207,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.714.1" +version = "1.715.0" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -15200,7 +15218,7 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "async-recursion", @@ -15230,7 +15248,7 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "async-trait", @@ -15254,7 +15272,7 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "async-trait", @@ -15287,7 +15305,7 @@ dependencies = [ [[package]] name = "windmill-trigger-azure" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "async-trait", @@ -15320,7 +15338,7 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "async-trait", @@ -15340,7 +15358,7 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "async-trait", @@ -15374,7 +15392,7 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "async-trait", @@ -15410,7 +15428,7 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "async-trait", @@ -15433,7 +15451,7 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "async-trait", @@ -15457,7 +15475,7 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "async-nats", @@ -15481,7 +15499,7 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "async-trait", @@ -15516,7 +15534,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "async-trait", @@ -15544,7 +15562,7 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "async-trait", @@ -15569,7 +15587,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "bitflags 2.12.1", @@ -15588,7 +15606,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "async-once-cell", @@ -15698,7 +15716,7 @@ dependencies = [ [[package]] name = "windmill-worker-volumes" -version = "1.714.1" +version = "1.715.0" dependencies = [ "bytes", "futures", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index addefff0cc..b86c4f1931 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.714.1" +version = "1.715.0" authors.workspace = true edition.workspace = true @@ -87,7 +87,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"] [workspace.package] -version = "1.714.1" +version = "1.715.0" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/parsers/windmill-parser-wasm/Cargo.lock b/backend/parsers/windmill-parser-wasm/Cargo.lock index 237a29168c..ef6745521b 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.lock +++ b/backend/parsers/windmill-parser-wasm/Cargo.lock @@ -6183,7 +6183,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill-common" -version = "1.714.1" +version = "1.715.0" dependencies = [ "aho-corasick", "anyhow", @@ -6263,7 +6263,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.714.1" +version = "1.715.0" dependencies = [ "proc-macro2", "quote", @@ -6275,7 +6275,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.714.1" +version = "1.715.0" dependencies = [ "convert_case", "serde", @@ -6284,7 +6284,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "lazy_static", @@ -6296,7 +6296,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "serde_json", @@ -6308,7 +6308,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "gosyn", @@ -6320,7 +6320,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "lazy_static", @@ -6332,7 +6332,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "serde_json", @@ -6344,7 +6344,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "nu-parser", @@ -6355,7 +6355,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6366,7 +6366,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6378,7 +6378,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6389,7 +6389,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "async-recursion", @@ -6411,7 +6411,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "serde_json", @@ -6423,7 +6423,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "lazy_static", @@ -6437,7 +6437,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "convert_case", @@ -6454,7 +6454,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "lazy_static", @@ -6467,7 +6467,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "serde", @@ -6479,7 +6479,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "lazy_static", @@ -6497,7 +6497,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -6513,7 +6513,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6529,7 +6529,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "getrandom 0.2.17", @@ -6561,7 +6561,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "serde", @@ -6572,7 +6572,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.714.1" +version = "1.715.0" dependencies = [ "anyhow", "bitflags", diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index a1b4bf831d..0aa6e5e8d4 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -12,7 +12,7 @@ resolver = "2" members = ["."] [workspace.package] -version = "1.714.1" +version = "1.715.0" edition = "2021" authors = ["Ruben Fiszel "] diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 13dcefea81..4687cc3a72 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.714.1 + version: 1.715.0 title: Windmill API contact: diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index f656bc4620..fd8f1a1036 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.714.1"; +export const VERSION = "v1.715.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/src/main.ts b/cli/src/main.ts index da11fcc575..2ae6db1be7 100755 --- a/cli/src/main.ts +++ b/cli/src/main.ts @@ -89,7 +89,7 @@ export { token, }; -export const VERSION = "1.714.1"; +export const VERSION = "1.715.0"; // Re-exported from constants.ts to maintain backwards compatibility export { WM_FORK_PREFIX } from "./core/constants.ts"; diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 82d56d2692..b61d57c0cd 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "@windmill-labs/components", - "version": "1.714.1", + "version": "1.715.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@windmill-labs/components", - "version": "1.714.1", + "version": "1.715.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { diff --git a/frontend/package.json b/frontend/package.json index e1b0a960ca..cb890fa2ef 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "@windmill-labs/components", - "version": "1.714.1", + "version": "1.715.0", "scripts": { "dev": "vite dev", "dev:ui-builder": "mv static/ui_builder static/ui_builder.dev-disabled 2>/dev/null || true ; trap 'mv static/ui_builder.dev-disabled static/ui_builder 2>/dev/null || true' EXIT ; vite dev", diff --git a/lsp/Pipfile b/lsp/Pipfile index 9bf5b1515a..cf8c250d40 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,7 +4,7 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.714.1" +wmill = ">=1.715.0" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index b58c43c88f..34e300f5f2 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: '3.0.3' info: - version: 1.714.1 + version: 1.715.0 title: OpenFlow Spec contact: name: Ruben Fiszel diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index 3d163e7def..6ff984ffa2 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. - ModuleVersion = '1.714.1' + ModuleVersion = '1.715.0' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index a81d545160..db829dad23 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.714.1" +version = "1.715.0" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index 842e4aa000..72c0224e8e 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.714.1", + "version": "1.715.0", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./sqlUtils.ts", "./client.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index d7d7b693a4..d4cac9d01b 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.714.1", + "version": "1.715.0", "author": "Ruben Fiszel", "license": "Apache 2.0", "sideEffects": false, diff --git a/version.txt b/version.txt index f51afc665c..24c842d87a 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.714.1 +1.715.0 From 0ba128afe797bd016da60563949ac3abbbfe1978 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 3 Jun 2026 12:31:45 +0200 Subject: [PATCH 26/61] fix(security): scope variable and resource value caches by caller identity (#9427) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The variable and resource value caches (backing `GET /api/w/{w}/variables/get_value/{path}?allow_cache=true` and `.../resources/get_value_interpolated/{path}?allow_cache=true`) are consulted before the per-folder RLS query and store the already-decrypted value. The resource cache was keyed only by `workspace:path` with no caller identity, so a cache entry warmed by a privileged peer using `allow_cache=true` could be returned to a caller with no access to the resource's folder on a cache hit within the 30s TTL — leaking another folder's decrypted secrets. Scope both caches to the caller's full authorization identity. The key is now `auth_identity(authed):workspace:path`, where `auth_identity` is a SHA-256 of the caller's effective authorization context (email, username, is_admin, is_operator, sorted groups, sorted folders, sorted scopes) — mirroring `job_read_access_cache_key`. Email alone is insufficient: the same email can resolve to different effective permissions via job/owner-scoped tokens, so a lower-privilege context must not reuse a higher-privilege context's entry. Job-context resource interpolation is handled correctly: only `$WM_*` contextual variables are resolved (and only when a `job_id` is present). The interpolation reports whether the value contains a `$WM_*` placeholder (`transform_json_value_tracked` + an `AtomicBool`). A value containing one is job-dependent — even on a no-job read where it's left unresolved — and is never cached (so a later job read never gets a stale placeholder or another job's context). Any value without a `$WM_*` placeholder is job-independent and cached under the identity key, shared across job contexts, so reads carrying a `job_id` still hit the cache. BEHAVIOR CHANGE: custom workspace environment variables are no longer interpolated into resource values via `$NAME` (this was undocumented and prevented caching of any `$`-prefixed value). Custom envs remain available to scripts/workers as before. Built-in `$WM_*` contextual variables in resource values are unchanged. The variable cache previously wrote with an identity-scoped key but read with the unscoped key, so it never hit (a latent functional bug that happened to be safe). Aligning the read path enables the cache and makes it identity-scoped by construction. Secret variables are cached too, but the entry carries the `is_secret` flag so a cache hit re-runs the per-read side effects a secret read performs — the EE `variables.decrypt_secret` audit and running-job secret registration (factored into `audit_decrypt_secret`, shared by both paths). The unused `invalidate_{variable,resource}_cache` helpers can no longer target identity-scoped entries; documented the constraint and refreshed the stale key-format docs on the cache statics. Tests: - integration regression for both caches: a folder-scoped user warms the cache via allow_cache=true, then a user without folder access is denied (401) and never receives the cached value. - integration regression that variables (secret included) are served from cache. - integration regression for job context: plain and non-`$WM_` `$`-string resources stay cached and are served under a job_id, while a `$WM_*` resource (warmed without a job_id) is not cached. - unit tests for `auth_identity`. Co-authored-by: Claude Opus 4.8 (1M context) --- backend/Cargo.lock | 2 + .../tests/fixtures/resource_cache_rls.sql | 23 ++ .../tests/fixtures/variable_cache_rls.sql | 15 ++ .../tests/resources.rs | 111 +++++++++ .../tests/variables.rs | 98 +++++++- backend/windmill-store/Cargo.toml | 2 + backend/windmill-store/src/resources.rs | 94 ++++++-- .../windmill-store/src/var_resource_cache.rs | 227 ++++++++++++++++-- backend/windmill-store/src/variables.rs | 77 ++++-- 9 files changed, 591 insertions(+), 58 deletions(-) create mode 100644 backend/windmill-api-integration-tests/tests/fixtures/resource_cache_rls.sql create mode 100644 backend/windmill-api-integration-tests/tests/fixtures/variable_cache_rls.sql diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 567af4bbcc..9d890d9b05 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -15225,6 +15225,7 @@ dependencies = [ "axum 0.8.9", "chrono", "futures", + "hex", "http 1.4.1", "hyper 1.10.1", "lazy_static", @@ -15232,6 +15233,7 @@ dependencies = [ "reqwest 0.13.1", "serde", "serde_json", + "sha2 0.10.9", "sql-builder", "sqlx", "tokio", diff --git a/backend/windmill-api-integration-tests/tests/fixtures/resource_cache_rls.sql b/backend/windmill-api-integration-tests/tests/fixtures/resource_cache_rls.sql new file mode 100644 index 0000000000..8c30eab5b2 --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/fixtures/resource_cache_rls.sql @@ -0,0 +1,23 @@ +-- Fixture for the resource-value interpolation cache RLS regression test. +-- Extends base.sql (which defines test-user [admin], test-user-2, test-user-3 +-- and their tokens). +-- +-- A folder `secret` is readable ONLY by test-user-2 (via extra_perms). It holds a +-- variable and a resource that interpolates it. test-user-3 has no access to the +-- folder, so a cache entry warmed by test-user-2 with allow_cache=true must never +-- be served back to test-user-3. + +INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms, created_by) +VALUES ('test-workspace', 'secret', 'Secret Folder', '{}', + '{"u/test-user-2": true}', 'test-user'); + +-- A (non-secret) variable gated to the `secret` folder; its value gets interpolated +-- into the resource value below and ends up in the cached, already-resolved blob. +INSERT INTO variable (workspace_id, path, value, is_secret, description, extra_perms) +VALUES ('test-workspace', 'f/secret/db_password', 'LEAKED_FOLDER_SECRET', false, + 'Folder-gated secret', '{}'); + +INSERT INTO resource (workspace_id, path, value, description, resource_type, extra_perms, created_by) +VALUES ('test-workspace', 'f/secret/cache_target', + '{"host": "db.internal", "password": "$var:f/secret/db_password"}', + 'Folder-gated resource referencing a folder-gated variable', 'object', '{}', 'test-user'); diff --git a/backend/windmill-api-integration-tests/tests/fixtures/variable_cache_rls.sql b/backend/windmill-api-integration-tests/tests/fixtures/variable_cache_rls.sql new file mode 100644 index 0000000000..69a6810b7b --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/fixtures/variable_cache_rls.sql @@ -0,0 +1,15 @@ +-- Fixture for the variable-value cache RLS regression test. +-- Extends base.sql (which defines test-user [admin], test-user-2, test-user-3 +-- and their tokens). +-- +-- A folder `secret` is readable ONLY by test-user-2 (via extra_perms). It holds a +-- variable that test-user-2 can read but test-user-3 cannot. A cache entry warmed +-- by test-user-2 with allow_cache=true must never be served back to test-user-3. + +INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms, created_by) +VALUES ('test-workspace', 'secret', 'Secret Folder', '{}', + '{"u/test-user-2": true}', 'test-user'); + +INSERT INTO variable (workspace_id, path, value, is_secret, description, extra_perms) +VALUES ('test-workspace', 'f/secret/cache_target_var', 'LEAKED_VAR_SECRET', false, + 'Folder-gated variable', '{}'); diff --git a/backend/windmill-api-integration-tests/tests/resources.rs b/backend/windmill-api-integration-tests/tests/resources.rs index 363217712f..cc78056176 100644 --- a/backend/windmill-api-integration-tests/tests/resources.rs +++ b/backend/windmill-api-integration-tests/tests/resources.rs @@ -477,6 +477,117 @@ async fn test_resource_endpoints(db: Pool) -> anyhow::Result<()> { Ok(()) } +/// Regression test: the resource-value interpolation cache +/// (`get_value_interpolated?allow_cache=true`) must be identity-scoped. test-user-2 +/// (folder access) warms the cache; test-user-3 (no access) must then be denied rather +/// than served the cached, already-decrypted value. Pre-fix the unscoped key returned +/// a 200 with the secret here. +#[sqlx::test(migrations = "../migrations", fixtures("base", "resource_cache_rls"))] +async fn test_resource_value_cache_is_identity_scoped(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let url = format!( + "{}?allow_cache=true", + resource_url(port, "get_value_interpolated", "f/secret/cache_target") + ); + let get = |token: &str| { + client() + .get(url.as_str()) + .header("Authorization", format!("Bearer {token}")) + }; + + // test-user-2 has folder access and WARMS the cache. + let resp = get("SECRET_TOKEN_2").send().await?; + assert_eq!(resp.status(), 200); + assert!(resp.text().await?.contains("LEAKED_FOLDER_SECRET")); + + // test-user-3 has no folder access: must miss the cache and be denied (401), not leak. + let resp = get("SECRET_TOKEN_3").send().await?; + assert_eq!(resp.status(), 401); + assert!(!resp.text().await?.contains("LEAKED_FOLDER_SECRET")); + + Ok(()) +} + +/// A resource whose value contains a `$WM_*` contextual variable (e.g. `$WM_TOKEN`) is +/// job-dependent and must NEVER be cached — even when first read WITHOUT a `job_id`, where the +/// placeholder is left unresolved (caching that would serve a stale placeholder to a later job +/// read). Any other value — plain, or a non-`$WM_` `$`-string like `$HOME` (which is NOT +/// interpolated, so it's constant) — is job-independent and IS cached, with the entry shared +/// across job contexts (a read carrying a `job_id` still hits it, keeping the hit ratio up). +/// We prove all three by warming each (no job_id), deleting the row directly (cache survives), +/// then re-reading: the job-independent ones are still served from cache — even under a +/// `job_id` — while the `$WM_*` one was never cached and 404s. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_resource_cache_handles_job_context(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/resources"); + + let plain = "u/test-user/plain_res"; + let dollar = "u/test-user/dollar_res"; // non-$WM_ `$`-string: not interpolated, cacheable + let jobctx = "u/test-user/jobctx_res"; + for (path, value) in [ + (plain, json!({"v": 1})), + (dollar, json!({"d": "$HOME"})), + (jobctx, json!({"j": "$WM_JOB_ID"})), + ] { + let resp = authed(client().post(format!("{base}/create"))) + .json( + &json!({ "path": path, "value": value, "description": "", "resource_type": "object" }), + ) + .send() + .await?; + assert_eq!(resp.status(), 201); + } + + let get = |path: &str, query: &str| { + let url = format!("{base}/get_value_interpolated/{path}?{query}"); + async move { authed(client().get(url)).send().await.unwrap() } + }; + + // Warm all three WITHOUT a job context (the placeholder is left unresolved for `jobctx`). + for path in [plain, dollar, jobctx] { + assert_eq!(get(path, "allow_cache=true").await.status(), 200); + } + + // Delete the rows directly — bypasses the API/NOTIFY, so the in-memory cache survives. + for path in [plain, dollar, jobctx] { + sqlx::query("DELETE FROM resource WHERE workspace_id = 'test-workspace' AND path = $1") + .bind(path) + .execute(&db) + .await?; + } + + // Job-independent values are cached and still served even under a job_id (a random uuid is + // fine: a cache hit short-circuits before any job lookup). `$HOME` is a non-`$WM_` string, + // so it's not interpolated and stays cacheable. + for path in [plain, dollar] { + let resp = get( + path, + "allow_cache=true&job_id=11111111-1111-4111-8111-111111111111", + ) + .await; + assert_eq!( + resp.status(), + 200, + "job-independent resource ({path}) must stay cached and be served under a job_id" + ); + } + + // The `$WM_*` resource was never cached → the (now deleted) row is not found. + let resp = get(jobctx, "allow_cache=true").await; + assert_ne!( + resp.status(), + 200, + "resource with a $WM_* contextual variable must not be cached" + ); + + Ok(()) +} + #[cfg(feature = "mcp")] #[sqlx::test(migrations = "../migrations", fixtures("base", "resources_test"))] async fn test_mcp_tools(db: Pool) -> anyhow::Result<()> { diff --git a/backend/windmill-api-integration-tests/tests/variables.rs b/backend/windmill-api-integration-tests/tests/variables.rs index 0d4edaff91..e5018f4f97 100644 --- a/backend/windmill-api-integration-tests/tests/variables.rs +++ b/backend/windmill-api-integration-tests/tests/variables.rs @@ -108,12 +108,10 @@ async fn test_variable_endpoints(db: Pool) -> anyhow::Result<()> { assert_eq!(secret["value"], serde_json::Value::Null); // list with path_start filter - let resp = authed(client().get(format!( - "{base}/list?path_start=u/test-user/plain" - ))) - .send() - .await - .unwrap(); + let resp = authed(client().get(format!("{base}/list?path_start=u/test-user/plain"))) + .send() + .await + .unwrap(); assert_eq!(resp.status(), 200); let list = resp.json::>().await?; assert_eq!(list.len(), 1); @@ -252,3 +250,91 @@ async fn test_variable_endpoints(db: Pool) -> anyhow::Result<()> { Ok(()) } + +/// Regression test: the variable-value cache (`get_value?allow_cache=true`) must be +/// identity-scoped. test-user-2 (folder access) warms the cache; test-user-3 (no access) +/// must then be denied rather than served the cached value. +#[sqlx::test(migrations = "../migrations", fixtures("base", "variable_cache_rls"))] +async fn test_variable_value_cache_is_identity_scoped(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let url = format!( + "{}?allow_cache=true", + variable_url(port, "get_value", "f/secret/cache_target_var") + ); + let get = |token: &str| { + client() + .get(url.as_str()) + .header("Authorization", format!("Bearer {token}")) + }; + + // test-user-2 has folder access and WARMS the cache. + let resp = get("SECRET_TOKEN_2").send().await?; + assert_eq!(resp.status(), 200); + assert!(resp.text().await?.contains("LEAKED_VAR_SECRET")); + + // test-user-3 has no folder access: must miss the cache and be denied (401), not leak. + let resp = get("SECRET_TOKEN_3").send().await?; + assert_eq!(resp.status(), 401); + assert!(!resp.text().await?.contains("LEAKED_VAR_SECRET")); + + Ok(()) +} + +/// Secret variables ARE cached (with their per-read side effects — the EE +/// `variables.decrypt_secret` audit and running-job secret registration — re-run on every +/// hit; that re-emission is not observable in the OSS build since `audit_log` is a no-op). +/// We assert the caching itself: warm the cache, delete the row directly (no API/NOTIFY, so +/// the in-memory cache survives), and re-read with `allow_cache=true` — the value is still +/// returned from cache. A non-secret variable behaves identically (control). +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_variables_are_cached(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/variables"); + + let plain = "u/test-user/cache_plain_probe"; + let secret = "u/test-user/cache_secret_probe"; + + // Create one non-secret and one secret variable (the secret is stored encrypted). + for (path, value, is_secret) in [ + (plain, "PLAIN_PROBE", false), + (secret, "SECRET_PROBE", true), + ] { + let resp = authed(client().post(format!("{base}/create"))) + .json( + &json!({ "path": path, "value": value, "is_secret": is_secret, "description": "" }), + ) + .send() + .await?; + assert_eq!(resp.status(), 201); + } + + let read = |path: &str| { + let url = format!("{base}/get_value/{path}?allow_cache=true"); + async move { authed(client().get(url)).send().await.unwrap() } + }; + + // Warm the cache for both. + assert_eq!(read(plain).await.json::().await?, "PLAIN_PROBE"); + assert_eq!(read(secret).await.json::().await?, "SECRET_PROBE"); + + // Delete both rows directly — bypasses the API and its NOTIFY-based invalidation, so + // the in-memory cache survives. A subsequent read can only succeed from cache. + for path in [plain, secret] { + sqlx::query("DELETE FROM variable WHERE workspace_id = 'test-workspace' AND path = $1") + .bind(path) + .execute(&db) + .await?; + } + + // Both (secret included) are still served from the cache. + assert_eq!(read(plain).await.json::().await?, "PLAIN_PROBE"); + let resp = read(secret).await; + assert_eq!(resp.status(), 200, "secret must still be served from cache"); + assert_eq!(resp.json::().await?, "SECRET_PROBE"); + + Ok(()) +} diff --git a/backend/windmill-store/Cargo.toml b/backend/windmill-store/Cargo.toml index cb1e41ad05..b3aca5e669 100644 --- a/backend/windmill-store/Cargo.toml +++ b/backend/windmill-store/Cargo.toml @@ -45,6 +45,8 @@ tracing.workspace = true uuid.workspace = true quick_cache.workspace = true lazy_static.workspace = true +sha2.workspace = true +hex.workspace = true sql-builder.workspace = true async-recursion.workspace = true futures.workspace = true diff --git a/backend/windmill-store/src/resources.rs b/backend/windmill-store/src/resources.rs index 3e5292e805..402628e3ad 100644 --- a/backend/windmill-store/src/resources.rs +++ b/backend/windmill-store/src/resources.rs @@ -17,7 +17,7 @@ use windmill_common::db::DB; use windmill_common::workspaces::{check_deploy_rules, RuleCheckResult}; use crate::secret_backend_ext::rename_vault_secret; -use crate::var_resource_cache::{cache_resource, get_cached_resource}; +use crate::var_resource_cache::{auth_identity, cache_resource, get_cached_resource}; use windmill_common::utils::{escape_ilike_pattern, BulkDeleteRequest}; use windmill_common::webhook::{WebhookMessage, WebhookShared}; @@ -550,8 +550,18 @@ pub async fn get_resource_value_interpolated_internal<'a>( return Ok(Some(pg_creds)); } - if allow_cache { - if let Some(cached_value) = get_cached_resource(&workspace, &path) { + // Scope the cache to the caller's full authorization identity (not just email): the + // cached value is already decrypted/interpolated under this caller's RLS context, so it + // must never be served to a context that resolves to different permissions. Only + // job-independent values are ever stored (see the write below), so a hit is always safe + // to return regardless of the current `job_id`. + let cache_identity = allow_cache.then(|| match db_with_opt_authed.authed() { + Some(authed) => auth_identity(authed), + None => format!("\0system:{}", db_with_opt_authed.email()), + }); + + if let Some(identity) = cache_identity.as_deref() { + if let Some(cached_value) = get_cached_resource(&workspace, &path, identity) { return Ok(Some(cached_value)); } } @@ -575,17 +585,24 @@ pub async fn get_resource_value_interpolated_internal<'a>( let value = not_found_if_none(value_o, "Resource", path)?; if let Some(value) = value { - let r = transform_json_value( + // Track whether interpolation pulled in a `$WM_*` contextual variable. If it did, the + // result is job-dependent (and may embed `$WM_TOKEN`) and must not be cached; if not, + // it's job-independent and safe to cache and to serve to any job context. + let used_job_context = std::sync::atomic::AtomicBool::new(false); + let r = transform_json_value_tracked( &db_with_opt_authed, workspace, value, &job_id, token_for_context, 0, + &used_job_context, ) .await?; - if allow_cache { - cache_resource(&workspace, &path, r.clone()); + if let Some(identity) = cache_identity.as_deref() { + if !used_job_context.load(std::sync::atomic::Ordering::Relaxed) { + cache_resource(&workspace, &path, identity, r.clone()); + } } Ok(Some(r)) } else { @@ -601,14 +618,41 @@ pub async fn get_resource_value_interpolated_internal<'a>( // access could otherwise use to crash the API process. pub const MAX_RESOURCE_INTERPOLATION_DEPTH: u8 = 50; -#[async_recursion] pub async fn transform_json_value( - db_with_opt_authed: &DbWithOptAuthed, + db_with_opt_authed: &DbWithOptAuthed<'_, ApiAuthed>, workspace: &str, v: Value, job_id: &Option, token: Option<&str>, depth: u8, +) -> Result { + // Discard the job-context flag; callers that need it use `transform_json_value_tracked`. + let used_job_context = std::sync::atomic::AtomicBool::new(false); + transform_json_value_tracked( + db_with_opt_authed, + workspace, + v, + job_id, + token, + depth, + &used_job_context, + ) + .await +} + +/// Like [`transform_json_value`], but records into `used_job_context` whether the value +/// contains a `$WM_*` contextual variable (resolved from `job_id`/`token`). A value that did +/// not is job-independent and safe to cache; one that did must not be cached or shared across +/// jobs. +#[async_recursion] +pub async fn transform_json_value_tracked( + db_with_opt_authed: &DbWithOptAuthed<'_, ApiAuthed>, + workspace: &str, + v: Value, + job_id: &Option, + token: Option<&str>, + depth: u8, + used_job_context: &std::sync::atomic::AtomicBool, ) -> Result { if depth >= MAX_RESOURCE_INTERPOLATION_DEPTH { return Err(Error::internal_err(format!( @@ -652,15 +696,35 @@ pub async fn transform_json_value( tx.commit().await?; let v = not_found_if_none(v, "Resource", path)?; if let Some(v) = v { - transform_json_value(db_with_opt_authed, workspace, v, job_id, token, depth + 1) - .await + transform_json_value_tracked( + db_with_opt_authed, + workspace, + v, + job_id, + token, + depth + 1, + used_job_context, + ) + .await } else { Ok(Value::Null) } } - Value::String(y) if y.starts_with("$") && job_id.is_some() => { + // `$WM_*` is the reserved contextual-variable namespace (`$WM_TOKEN`, `$WM_JOB_ID`, + // ...); its resolved value depends on the job, so a value containing one is + // job-dependent and must never be cached — including on a no-job read, where the + // placeholder is left unresolved (caching it would then serve a stale placeholder to a + // later job read). Any other `$...` string (custom workspace envs, `$5.00`, `$HOME`, jq + // paths) is NOT interpolated here — it resolves to itself regardless of context and so + // stays cacheable (handled by the catch-all below). Note: custom workspace envs are + // intentionally not resolved inside resource values (they remain available to scripts). + Value::String(y) if y.starts_with("$WM_") => { + used_job_context.store(true, std::sync::atomic::Ordering::Relaxed); + let Some(job_id) = *job_id else { + // No job context to resolve against; leave the placeholder unchanged. + return Ok(Value::String(y)); + }; let mut tx = db_with_opt_authed.begin().await?; - let job_id = job_id.unwrap(); let job = sqlx::query!( "SELECT v2_job.permissioned_as_email, @@ -731,13 +795,14 @@ pub async fn transform_json_value( Value::Array(mut arr) if depth <= 2 && arr.len() <= 1000 => { for i in 0..arr.len() { let val = std::mem::take(&mut arr[i]); - arr[i] = transform_json_value( + arr[i] = transform_json_value_tracked( db_with_opt_authed, workspace, val, job_id, token, depth + 1, + used_job_context, ) .await?; } @@ -754,13 +819,14 @@ pub async fn transform_json_value( } Value::Object(mut m) => { for (a, b) in m.clone().into_iter() { - let v = transform_json_value( + let v = transform_json_value_tracked( db_with_opt_authed, workspace, b, job_id, token, depth + 1, + used_job_context, ) .await?; m.insert(a.clone(), v); diff --git a/backend/windmill-store/src/var_resource_cache.rs b/backend/windmill-store/src/var_resource_cache.rs index f7ce2aeecf..3e89f8579e 100644 --- a/backend/windmill-store/src/var_resource_cache.rs +++ b/backend/windmill-store/src/var_resource_cache.rs @@ -8,7 +8,9 @@ use quick_cache::sync::Cache; use serde_json::Value; +use sha2::{Digest, Sha256}; use std::time::{SystemTime, UNIX_EPOCH}; +use windmill_common::db::Authable; /// Cache TTL for variables and resources (30seconds) const CACHE_TTL_SECS: u64 = 30; @@ -40,11 +42,23 @@ impl CacheEntry { } } -lazy_static::lazy_static! { - /// Cache for individual variable values: key = "workspace_id:path" - pub static ref VARIABLE_CACHE: Cache> = Cache::new(1000); +/// A cached variable value plus whether it is a secret. `is_secret` is retained so a +/// cache hit can re-run the per-read side effects of a secret read (the +/// `variables.decrypt_secret` audit and running-job secret registration) that the +/// original miss performed — a hit must be observably equivalent to a miss. +#[derive(Clone, Debug)] +pub struct CachedVariable { + pub value: String, + pub is_secret: bool, +} - /// Cache for resource values: key = "workspace_id:path" +lazy_static::lazy_static! { + /// Cache for individual variable values. Key: [`identity_cache_key`] + /// (`identity:workspace_id:path`) — scoped to the caller's authorization context. + pub static ref VARIABLE_CACHE: Cache> = Cache::new(1000); + + /// Cache for interpolated resource values. Key: [`identity_cache_key`] + /// (`identity:workspace_id:path`) — scoped to the caller's authorization context. pub static ref RESOURCE_CACHE: Cache> = Cache::new(1000); } @@ -53,9 +67,73 @@ pub fn cache_key(workspace_id: &str, path: &str) -> String { format!("{}:{}", workspace_id, path) } -/// Get cached variable if available and not expired -pub fn get_cached_variable(workspace_id: &str, path: &str) -> Option { - let key = cache_key(workspace_id, path); +/// Hash the caller's full authorization context into a stable identity string. +/// +/// Email alone is **not** a sufficient scope: the same email can resolve to different +/// effective permissions (`username`, groups, folders, scopes, admin/operator) through +/// job- or owner-scoped tokens that share an email but carry a narrower `permissioned_as`. +/// Every input that determines what the caller may read is folded in, mirroring +/// `job_read_access_cache_key` in windmill-api, so a lower-privilege context can never +/// reuse a higher-privilege context's cache entry. Variable-length fields are +/// length-prefixed to keep the encoding injective. +pub fn auth_identity(authed: &A) -> String { + let mut hasher = Sha256::new(); + let field = |hasher: &mut Sha256, bytes: &[u8]| { + hasher.update((bytes.len() as u32).to_be_bytes()); + hasher.update(bytes); + }; + hasher.update([authed.is_admin() as u8, authed.is_operator() as u8]); + field(&mut hasher, authed.email().as_bytes()); + field(&mut hasher, authed.username().as_bytes()); + let mut groups: Vec<&str> = authed.groups().iter().map(String::as_str).collect(); + groups.sort_unstable(); + hasher.update((groups.len() as u32).to_be_bytes()); + for g in groups { + field(&mut hasher, g.as_bytes()); + } + let mut folders: Vec<&str> = authed.folders().iter().map(|f| f.0.as_str()).collect(); + folders.sort_unstable(); + hasher.update((folders.len() as u32).to_be_bytes()); + for f in folders { + field(&mut hasher, f.as_bytes()); + } + match authed.scopes() { + // u32::MAX length-prefix marks "no scopes" so it can't collide with an empty list. + None => hasher.update(u32::MAX.to_be_bytes()), + Some(scopes) => { + let mut scopes: Vec<&str> = scopes.iter().map(String::as_str).collect(); + scopes.sort_unstable(); + hasher.update((scopes.len() as u32).to_be_bytes()); + for s in scopes { + field(&mut hasher, s.as_bytes()); + } + } + } + hex::encode(hasher.finalize()) +} + +/// Generate an identity-scoped cache key (`identity:workspace_id:path`). +/// +/// Both the variable and resource caches store *already-decrypted* values that were +/// resolved under the caller's row-level-security context. The cache is consulted before +/// the per-folder RLS query runs, so an unscoped `workspace:path` key would let an entry +/// warmed by one caller (via `allow_cache=true`) be served to a different caller who has +/// no access to the underlying folder, leaking decrypted secrets within the TTL. `identity` +/// is [`auth_identity`] — the hash of the caller's full authorization context — so a hit +/// can only ever be returned to a caller whose authorized read populated it. +fn identity_cache_key(identity: &str, workspace_id: &str, path: &str) -> String { + format!("{}:{}", identity, cache_key(workspace_id, path)) +} + +/// Get cached variable if available and not expired. Scoped to `identity` +/// ([`auth_identity`]); see [`identity_cache_key`]. Returns the value and its `is_secret` +/// flag so the caller can re-run a secret read's side effects on a hit. +pub fn get_cached_variable( + workspace_id: &str, + path: &str, + identity: &str, +) -> Option { + let key = identity_cache_key(identity, workspace_id, path); VARIABLE_CACHE.get(&key).and_then(|entry| { if entry.is_expired() { VARIABLE_CACHE.remove(&key); @@ -67,17 +145,21 @@ pub fn get_cached_variable(workspace_id: &str, path: &str) -> Option { }) } -/// Cache variable data -pub fn cache_variable(workspace_id: &str, path: &str, email: &str, variable: String) { - let key = format!("{}:{}", email, cache_key(workspace_id, path)); +/// Cache variable data, scoped to the caller identity. See [`get_cached_variable`]. +pub fn cache_variable(workspace_id: &str, path: &str, identity: &str, variable: CachedVariable) { + let key = identity_cache_key(identity, workspace_id, path); let entry = CacheEntry::new(variable); VARIABLE_CACHE.insert(key.clone(), entry); tracing::debug!("Cached variable {}", key); } -/// Get cached resource if available and not expired -pub fn get_cached_resource(workspace_id: &str, path: &str) -> Option { - let key = cache_key(workspace_id, path); +/// Get cached resource if available and not expired. +/// +/// Scoped to `identity` ([`auth_identity`]); see [`identity_cache_key`]. The cached value +/// is the *already-interpolated* resource — its `$var:`/`$res:` secrets are resolved and +/// decrypted inline — so it must never cross authorization boundaries. +pub fn get_cached_resource(workspace_id: &str, path: &str, identity: &str) -> Option { + let key = identity_cache_key(identity, workspace_id, path); RESOURCE_CACHE.get(&key).and_then(|entry| { if entry.is_expired() { RESOURCE_CACHE.remove(&key); @@ -89,22 +171,28 @@ pub fn get_cached_resource(workspace_id: &str, path: &str) -> Option { }) } -/// Cache resource data -pub fn cache_resource(workspace_id: &str, path: &str, resource: Value) { - let key = cache_key(workspace_id, path); +/// Cache resource data, scoped to the caller identity. See [`get_cached_resource`]. +pub fn cache_resource(workspace_id: &str, path: &str, identity: &str, resource: Value) { + let key = identity_cache_key(identity, workspace_id, path); let entry = CacheEntry::new(resource); RESOURCE_CACHE.insert(key.clone(), entry); tracing::debug!("Cached resource {}", key); } -/// Invalidate specific variable from cache +/// Invalidate a variable from the cache. +/// +/// NOTE: entries are keyed by [`identity_cache_key`] (`identity:workspace:path`), so this +/// `workspace:path` key cannot target them — it only removes a legacy unscoped entry, if +/// any. Per-identity entries are not enumerable here; rely on the 30s TTL for staleness, +/// or use [`clear_all_caches`] to force a full flush. Currently unused. pub fn invalidate_variable_cache(workspace_id: &str, path: &str) { let key = cache_key(workspace_id, path); VARIABLE_CACHE.remove(&key); tracing::info!("Variable cache invalidated for {}", key); } -/// Invalidate specific resource from cache +/// Invalidate a resource from the cache. Same identity-scoping caveat as +/// [`invalidate_variable_cache`]. Currently unused. pub fn invalidate_resource_cache(workspace_id: &str, path: &str) { let key = cache_key(workspace_id, path); RESOURCE_CACHE.remove(&key); @@ -118,3 +206,106 @@ pub fn clear_all_caches() { RESOURCE_CACHE.clear(); tracing::debug!("All variable/resource caches cleared"); } + +#[cfg(test)] +mod tests { + use super::*; + + /// Minimal [`Authable`] double so we can assert which authorization fields the + /// cache identity is sensitive to, without standing up a full auth stack. + struct FakeAuthed { + email: String, + username: String, + is_admin: bool, + is_operator: bool, + groups: Vec, + folders: Vec<(String, bool, bool)>, + scopes: Option>, + } + + impl FakeAuthed { + fn base() -> Self { + Self { + email: "alice@x.dev".to_string(), + username: "alice".to_string(), + is_admin: false, + is_operator: false, + groups: vec!["all".to_string()], + folders: vec![("shared".to_string(), false, false)], + scopes: None, + } + } + } + + impl Authable for FakeAuthed { + fn email(&self) -> &str { + &self.email + } + fn username(&self) -> &str { + &self.username + } + fn is_admin(&self) -> bool { + self.is_admin + } + fn is_operator(&self) -> bool { + self.is_operator + } + fn groups(&self) -> &[String] { + &self.groups + } + fn folders(&self) -> &[(String, bool, bool)] { + &self.folders + } + fn scopes(&self) -> Option<&[String]> { + self.scopes.as_deref() + } + } + + // Email alone must NOT determine the cache identity: two contexts that share an email + // but resolve to different effective permissions must get distinct identities, so a + // lower-privilege context can never reuse a higher-privilege one's cached secret. + #[test] + fn auth_identity_is_not_just_email() { + let base = auth_identity(&FakeAuthed::base()); + + let mut more_folders = FakeAuthed::base(); + more_folders + .folders + .push(("secret".to_string(), false, false)); + assert_ne!(base, auth_identity(&more_folders), "folders must matter"); + + let mut more_groups = FakeAuthed::base(); + more_groups.groups.push(("devs").to_string()); + assert_ne!(base, auth_identity(&more_groups), "groups must matter"); + + let mut other_user = FakeAuthed::base(); + other_user.username = "bob".to_string(); + assert_ne!(base, auth_identity(&other_user), "username must matter"); + + let mut admin = FakeAuthed::base(); + admin.is_admin = true; + assert_ne!(base, auth_identity(&admin), "is_admin must matter"); + + let mut operator = FakeAuthed::base(); + operator.is_operator = true; + assert_ne!(base, auth_identity(&operator), "is_operator must matter"); + + let mut scoped = FakeAuthed::base(); + scoped.scopes = Some(vec!["resources:read:f/secret/x".to_string()]); + assert_ne!(base, auth_identity(&scoped), "scopes must matter"); + } + + // Identical authorization contexts must produce the same identity (so the same caller + // gets a cache hit), and ordering of groups/folders must not change the identity. + #[test] + fn auth_identity_is_stable_and_order_independent() { + let a = FakeAuthed::base(); + assert_eq!(auth_identity(&a), auth_identity(&FakeAuthed::base())); + + let mut reordered = FakeAuthed::base(); + reordered.groups = vec!["all".to_string(), "devs".to_string()]; + let mut other_order = FakeAuthed::base(); + other_order.groups = vec!["devs".to_string(), "all".to_string()]; + assert_eq!(auth_identity(&reordered), auth_identity(&other_order)); + } +} diff --git a/backend/windmill-store/src/variables.rs b/backend/windmill-store/src/variables.rs index 2d767b393b..24739ce940 100644 --- a/backend/windmill-store/src/variables.rs +++ b/backend/windmill-store/src/variables.rs @@ -42,7 +42,9 @@ use windmill_common::{ worker::CLOUD_HOSTED, }; -use crate::var_resource_cache::{cache_variable, get_cached_variable}; +use crate::var_resource_cache::{ + auth_identity, cache_variable, get_cached_variable, CachedVariable, +}; use lazy_static::lazy_static; use serde::Deserialize; use sqlx::{Acquire, Postgres, Transaction}; @@ -1204,15 +1206,55 @@ fn replace_path(v: serde_json::Value, path: &str, npath: &str) -> Value { } } +/// Emit the `variables.decrypt_secret` audit event for a secret-variable read. Run on both +/// the cache-miss and cache-hit paths so `allow_cache` never skips secret-access auditing. +async fn audit_decrypt_secret( + db_with_opt_authed: &DbWithOptAuthed<'_, ApiAuthed>, + w_id: &str, + path: &str, +) -> Result<()> { + let mut tx = db_with_opt_authed.db().begin().await?; + audit_log( + &mut *tx, + db_with_opt_authed, + "variables.decrypt_secret", + ActionKind::Execute, + w_id, + Some(path), + None, + ) + .await?; + tx.commit().await?; + Ok(()) +} + pub async fn get_value_internal<'a>( db_with_opt_authed: &'a DbWithOptAuthed<'a, ApiAuthed>, w_id: &str, path: &str, allow_cache: bool, ) -> Result { - if allow_cache { - if let Some(cached_variable) = get_cached_variable(&w_id, &path) { - return Ok(cached_variable); + // Scope the cache to the caller's full authorization identity (not just email): the + // cached value is the decrypted variable, resolved under this caller's RLS context. + let cache_identity = allow_cache.then(|| match db_with_opt_authed.authed() { + Some(authed) => auth_identity(authed), + None => format!("\0system:{}", db_with_opt_authed.email()), + }); + + if let Some(identity) = cache_identity.as_deref() { + if let Some(cached) = get_cached_variable(&w_id, &path, identity) { + // A cache hit must be observably equivalent to a miss: re-run the per-read side + // effects a secret read performs (the `variables.decrypt_secret` audit and + // running-job secret registration) so `allow_cache` never silently skips them. + if cached.is_secret { + audit_decrypt_secret(db_with_opt_authed, &w_id, &path).await?; + if !cached.value.is_empty() { + windmill_common::sensitive_log_masks::register_secret_for_all_running_jobs( + &cached.value, + ); + } + } + return Ok(cached.value); } } @@ -1234,19 +1276,7 @@ pub async fn get_value_internal<'a>( }; let r = if variable.is_secret { - // let audit_author = - let mut tx = db_with_opt_authed.db().begin().await?; - audit_log( - &mut *tx, - db_with_opt_authed, - "variables.decrypt_secret", - ActionKind::Execute, - &w_id, - Some(&variable.path), - None, - ) - .await?; - tx.commit().await?; + audit_decrypt_secret(db_with_opt_authed, &w_id, &variable.path).await?; let value = variable.value; if variable.is_expired.unwrap_or(false) && variable.account.is_some() { @@ -1282,9 +1312,16 @@ pub async fn get_value_internal<'a>( windmill_common::sensitive_log_masks::register_secret_for_all_running_jobs(&r); } - // Cache the result when explicitly allowed and caching appropriate - if allow_cache { - cache_variable(&w_id, &path, db_with_opt_authed.email(), r.clone()); + // Cache the result when explicitly allowed. Secrets are cached too: their per-read side + // effects (audit + running-job registration) are re-run on a hit (see the hit path above), + // and `is_secret` is stored so the hit knows to do so. + if let Some(identity) = cache_identity.as_deref() { + cache_variable( + &w_id, + &path, + identity, + CachedVariable { value: r.clone(), is_secret: variable.is_secret }, + ); } Ok(r) From cf5fefb521479170b9dc64b884630c4dac789931 Mon Sep 17 00:00:00 2001 From: centdix <40307056+centdix@users.noreply.github.com> Date: Wed, 3 Jun 2026 12:33:16 +0200 Subject: [PATCH 27/61] feat: add metadata generation model setting (#9418) --- .../tests/workspaces.rs | 8 +- .../windmill-api-workspaces/src/workspaces.rs | 3 + backend/windmill-api/openapi-deref.json | 97 ++++++++++++++++- backend/windmill-api/openapi-deref.yaml | 102 +++++++++++++++++- backend/windmill-api/openapi.yaml | 4 + backend/windmill-api/src/ai.rs | 2 + frontend/src/lib/aiStore.ts | 13 +++ .../lib/components/copilot/MetadataGen.svelte | 6 +- .../copilot/chat/openai-responses.ts | 20 ++-- frontend/src/lib/components/copilot/lib.ts | 5 +- .../workspaceSettings/AISettings.svelte | 38 +++++++ .../InstanceFallbackSettings.svelte | 16 ++- 12 files changed, 299 insertions(+), 15 deletions(-) diff --git a/backend/windmill-api-integration-tests/tests/workspaces.rs b/backend/windmill-api-integration-tests/tests/workspaces.rs index 131cfbbae5..3d208d4e25 100644 --- a/backend/windmill-api-integration-tests/tests/workspaces.rs +++ b/backend/windmill-api-integration-tests/tests/workspaces.rs @@ -709,7 +709,9 @@ async fn test_get_copilot_settings_state_reports_instance_ai_fallback_flags( "resource_path": "u/test-user/openai_instance", "models": ["gpt-4o-mini"] } - } + }, + "default_model": { "provider": "openai", "model": "gpt-4o-mini" }, + "metadata_model": { "provider": "openai", "model": "gpt-4o-mini" } }); let workspace_ai_config = json!({ "providers": { @@ -749,6 +751,10 @@ async fn test_get_copilot_settings_state_reports_instance_ai_fallback_flags( settings["instance_ai_summary"]["providers"][0]["models"][0], "gpt-4o-mini" ); + assert_eq!( + settings["instance_ai_summary"]["metadata_model"]["model"], + "gpt-4o-mini" + ); sqlx::query("UPDATE workspace_settings SET ai_config = $1 WHERE workspace_id = $2") .bind(workspace_ai_config) diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index db60ae7fac..00cebc32ff 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -340,6 +340,8 @@ pub struct InstanceAISummary { #[serde(skip_serializing_if = "Option::is_none")] pub default_model: Option, #[serde(skip_serializing_if = "Option::is_none")] + pub metadata_model: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub code_completion_model: Option, } @@ -825,6 +827,7 @@ pub fn build_instance_ai_summary(config: Option<&serde_json::Value>) -> Option- + Workspaces that reference this database via a ducklake + catalog or datatable database with resource_type + 'instance'. Computed at request time, not persisted. /settings/setup_custom_instance_pg_database/{name}: post: summary: >- @@ -4842,6 +4850,10 @@ paths: required: &ref_44 - model - provider + metadata_model: + type: object + properties: *ref_43 + required: *ref_44 code_completion_model: type: object properties: *ref_43 @@ -5828,6 +5840,10 @@ paths: type: object properties: *ref_43 required: *ref_44 + metadata_model: + type: object + properties: *ref_43 + required: *ref_44 code_completion_model: type: object properties: *ref_43 @@ -11364,6 +11380,13 @@ paths: description: >- If true, all steps run on the same worker for better performance + preserve_step_tags: + type: boolean + description: >- + If true and the flow runs on a custom worker tag, + steps that declare their own non-empty tag run on + it instead of inheriting the flow tag. Steps + without their own tag still inherit the flow tag. concurrent_limit: type: number description: >- @@ -12619,6 +12642,11 @@ paths: type: array items: type: string + skip_draft_deletion: + type: boolean + description: >- + When true (set by the CLI / git sync), deploying this script + does not delete an existing user draft at the same path. required: &ref_105 - path - summary @@ -15461,6 +15489,12 @@ paths: type: boolean deployment_message: type: string + skip_draft_deletion: + type: boolean + description: >- + When true (set by the CLI / git sync), deploying this + flow does not delete an existing user draft at the same + path. responses: '201': description: flow created @@ -15507,6 +15541,12 @@ paths: properties: deployment_message: type: string + skip_draft_deletion: + type: boolean + description: >- + When true (set by the CLI / git sync), deploying this + flow does not delete an existing user draft at the same + path. responses: '200': description: flow updated @@ -16244,6 +16284,11 @@ paths: type: array items: type: string + skip_draft_deletion: + type: boolean + description: >- + When true (set by the CLI / git sync), deploying this app + does not delete an existing user draft at the same path. required: - path - value @@ -16303,6 +16348,12 @@ paths: type: array items: type: string + skip_draft_deletion: + type: boolean + description: >- + When true (set by the CLI / git sync), deploying this + app does not delete an existing user draft at the same + path. required: - path - value @@ -16740,6 +16791,11 @@ paths: type: array items: type: string + skip_draft_deletion: + type: boolean + description: >- + When true (set by the CLI / git sync), deploying this app + does not delete an existing user draft at the same path. responses: '200': description: app updated @@ -16796,6 +16852,12 @@ paths: type: array items: type: string + skip_draft_deletion: + type: boolean + description: >- + When true (set by the CLI / git sync), deploying this + app does not delete an existing user draft at the same + path. js: type: string css: @@ -17954,6 +18016,13 @@ paths: description: >- If true, all steps run on the same worker for better performance + preserve_step_tags: + type: boolean + description: >- + If true and the flow runs on a custom worker tag, steps + that declare their own non-empty tag run on it instead + of inheriting the flow tag. Steps without their own tag + still inherit the flow tag. concurrent_limit: type: number description: Maximum number of concurrent executions of this flow @@ -30256,6 +30325,37 @@ paths: type: object additionalProperties: type: integer + /workers/workspace_fairness_events: + get: + summary: list last 100 workspace-fairness cap/uncap events (cloud-only) + operationId: getWorkspaceFairnessEvents + tags: + - worker + responses: + '200': + description: workspace fairness events (empty on non-cloud) + content: + application/json: + schema: + type: array + items: + type: object + properties: + timestamp: + type: string + format: date-time + operation: + type: string + workspace_id: + type: string + nullable: true + parameters: + type: object + nullable: true + additionalProperties: true + required: + - timestamp + - operation /configs/list_worker_groups: get: summary: list worker groups diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 4687cc3a72..7aeda79dfc 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -21569,6 +21569,8 @@ components: $ref: "#/components/schemas/AIProviderConfig" default_model: $ref: "#/components/schemas/AIProviderModel" + metadata_model: + $ref: "#/components/schemas/AIProviderModel" code_completion_model: $ref: "#/components/schemas/AIProviderModel" custom_prompts: @@ -21604,6 +21606,8 @@ components: $ref: "#/components/schemas/InstanceAIProviderSummary" default_model: $ref: "#/components/schemas/AIProviderModel" + metadata_model: + $ref: "#/components/schemas/AIProviderModel" code_completion_model: $ref: "#/components/schemas/AIProviderModel" required: diff --git a/backend/windmill-api/src/ai.rs b/backend/windmill-api/src/ai.rs index 0e096eb798..fb92d28cb3 100644 --- a/backend/windmill-api/src/ai.rs +++ b/backend/windmill-api/src/ai.rs @@ -377,6 +377,8 @@ pub struct AIConfig { #[serde(skip_serializing_if = "Option::is_none")] pub default_model: Option, #[serde(skip_serializing_if = "Option::is_none")] + pub metadata_model: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub code_completion_model: Option, #[serde(skip_serializing_if = "Option::is_none")] pub custom_prompts: Option>, diff --git a/frontend/src/lib/aiStore.ts b/frontend/src/lib/aiStore.ts index 8abc303e18..f2bddb8874 100644 --- a/frontend/src/lib/aiStore.ts +++ b/frontend/src/lib/aiStore.ts @@ -21,6 +21,7 @@ export const copilotInfo = writable<{ enabled: boolean codeCompletionModel?: AIProviderModel defaultModel?: AIProviderModel + metadataModel?: AIProviderModel aiModels: AIProviderModel[] customPrompts?: Record maxTokensPerModel?: Record @@ -28,6 +29,7 @@ export const copilotInfo = writable<{ enabled: false, codeCompletionModel: undefined, defaultModel: undefined, + metadataModel: undefined, aiModels: [], customPrompts: {}, maxTokensPerModel: {} @@ -65,6 +67,7 @@ export function setCopilotInfo(aiConfig: AIConfig) { enabled: true, codeCompletionModel: aiConfig.code_completion_model, defaultModel: aiConfig.default_model, + metadataModel: aiConfig.metadata_model, aiModels: aiModels, customPrompts: aiConfig.custom_prompts ?? {}, maxTokensPerModel: aiConfig.max_tokens_per_model ?? {} @@ -76,6 +79,7 @@ export function setCopilotInfo(aiConfig: AIConfig) { enabled: false, codeCompletionModel: undefined, defaultModel: undefined, + metadataModel: undefined, aiModels: [], customPrompts: {}, maxTokensPerModel: {} @@ -92,6 +96,15 @@ export function getCurrentModel(): AIProviderModel { return model } +export function getMetadataModel(): AIProviderModel { + const info = get(copilotInfo) + const model = info.metadataModel ?? info.defaultModel ?? info.aiModels[0] + if (!model) { + throw new Error('No model selected') + } + return model +} + export function tryGetCurrentModel(): AIProviderModel | undefined { return get(copilotSessionModel) ?? get(copilotInfo).defaultModel ?? get(copilotInfo).aiModels[0] } diff --git a/frontend/src/lib/components/copilot/MetadataGen.svelte b/frontend/src/lib/components/copilot/MetadataGen.svelte index 1ef4b2ff08..1332ff1ce2 100644 --- a/frontend/src/lib/components/copilot/MetadataGen.svelte +++ b/frontend/src/lib/components/copilot/MetadataGen.svelte @@ -3,7 +3,7 @@ import { isInitialCode } from '$lib/script_helpers' import { Check, Loader2, Wand2 } from 'lucide-svelte' import { metadataCompletionEnabled } from '$lib/stores' - import { copilotInfo } from '$lib/aiStore' + import { copilotInfo, getMetadataModel } from '$lib/aiStore' import { onDestroy, untrack } from 'svelte' import { sendUserToast } from '$lib/toast' import { twMerge } from 'tailwind-merge' @@ -174,7 +174,9 @@ Generate a tool name for the script below: content: config.user.replace(`{${config.placeholderName}}`, placeholderContent) } ] - const response = await getCompletion(messages, abortController) + const response = await getCompletion(messages, abortController, undefined, { + forceModelProvider: getMetadataModel() + }) generatedContent = '' for await (const chunk of response) { generatedContent += getResponseFromEvent(chunk) diff --git a/frontend/src/lib/components/copilot/chat/openai-responses.ts b/frontend/src/lib/components/copilot/chat/openai-responses.ts index 378e1073f7..921b1cba45 100644 --- a/frontend/src/lib/components/copilot/chat/openai-responses.ts +++ b/frontend/src/lib/components/copilot/chat/openai-responses.ts @@ -14,10 +14,7 @@ import { import { processToolCall, type Tool, type ToolCallbacks } from './shared' import type { ResponseStream } from 'openai/lib/responses/ResponseStream.mjs' import type { AIProviderModel } from '$lib/gen' -import { - openAIResponsesUsageToChatTokenUsage, - type ChatTokenUsage -} from './tokenUsage' +import { openAIResponsesUsageToChatTokenUsage, type ChatTokenUsage } from './tokenUsage' interface ParsedCompletionResult { shouldContinue: boolean @@ -172,13 +169,22 @@ export async function getOpenAIResponsesCompletion( export async function* getOpenAIResponsesCompletionStream( messages: ChatCompletionMessageParam[], abortController: AbortController, - tools?: OpenAI.Chat.Completions.ChatCompletionTool[] + tools?: OpenAI.Chat.Completions.ChatCompletionTool[], + options?: { + forceModelProvider?: AIProviderModel + openaiClient?: OpenAI + } ): AsyncGenerator { - const { provider, config } = getProviderAndCompletionConfig({ messages, stream: true, tools }) + const { provider, config } = getProviderAndCompletionConfig({ + messages, + stream: true, + tools, + forceModelProvider: options?.forceModelProvider + }) const { instructions, input } = convertMessagesToResponsesInput(messages) const responsesConfig = convertCompletionConfigToResponsesConfig(config) - const openaiClient = workspaceAIClients.getOpenaiClient() + const openaiClient = options?.openaiClient ?? workspaceAIClients.getOpenaiClient() const runner = openaiClient.responses.stream( { diff --git a/frontend/src/lib/components/copilot/lib.ts b/frontend/src/lib/components/copilot/lib.ts index b87fb76ca6..16cbc961f4 100644 --- a/frontend/src/lib/components/copilot/lib.ts +++ b/frontend/src/lib/components/copilot/lib.ts @@ -895,7 +895,10 @@ export async function getCompletion( // Use Responses API for OpenAI and Azure OpenAI if ((provider === 'openai' || provider === 'azure_openai') && !options?.forceCompletions) { try { - const stream = getOpenAIResponsesCompletionStream(messages, abortController, tools) as any + const stream = getOpenAIResponsesCompletionStream(messages, abortController, tools, { + forceModelProvider: options?.forceModelProvider, + openaiClient: options?.openaiClient + }) as any return stream } catch (error) { console.error('Error using Responses API:', error) diff --git a/frontend/src/lib/components/workspaceSettings/AISettings.svelte b/frontend/src/lib/components/workspaceSettings/AISettings.svelte index 922492d6ac..ff1eeb9e9a 100644 --- a/frontend/src/lib/components/workspaceSettings/AISettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/AISettings.svelte @@ -68,6 +68,7 @@ let aiProviders: Exclude = $state({}) let codeCompletionModel: string | undefined = $state(undefined) let defaultModel: string | undefined = $state(undefined) + let metadataModel: string | undefined = $state(undefined) let customPrompts: Record = $state({}) let maxTokensPerModel: Record = $state({}) let usingOpenaiClientCredentialsOauth = $state(false) @@ -77,6 +78,7 @@ let initialAiProviders: Exclude = $state({}) let initialCodeCompletionModel: string | undefined = $state(undefined) let initialDefaultModel: string | undefined = $state(undefined) + let initialMetadataModel: string | undefined = $state(undefined) let initialCustomPrompts: Record = $state({}) let initialMaxTokensPerModel: Record = $state({}) let initialPrompts: Record = $state({}) @@ -89,6 +91,7 @@ function applyConfig(config: AIConfig | undefined) { aiProviders = clone(config?.providers ?? {}) defaultModel = config?.default_model?.model + metadataModel = config?.metadata_model?.model codeCompletionModel = config?.code_completion_model?.model customPrompts = clone(config?.custom_prompts ?? {}) maxTokensPerModel = clone(config?.max_tokens_per_model ?? {}) @@ -102,6 +105,7 @@ function storeInitialState() { initialAiProviders = clone(aiProviders) initialDefaultModel = defaultModel + initialMetadataModel = metadataModel initialCodeCompletionModel = codeCompletionModel initialCustomPrompts = clone(customPrompts) initialMaxTokensPerModel = clone(maxTokensPerModel) @@ -116,6 +120,7 @@ export function discard() { aiProviders = clone(initialAiProviders) defaultModel = initialDefaultModel + metadataModel = initialMetadataModel codeCompletionModel = initialCodeCompletionModel customPrompts = clone(initialCustomPrompts) maxTokensPerModel = clone(initialMaxTokensPerModel) @@ -149,6 +154,7 @@ let dirty = $derived( JSON.stringify(aiProviders) !== JSON.stringify(initialAiProviders) || defaultModel !== initialDefaultModel || + metadataModel !== initialMetadataModel || codeCompletionModel !== initialCodeCompletionModel || JSON.stringify(customPrompts) !== JSON.stringify(initialCustomPrompts) || JSON.stringify(maxTokensPerModel) !== JSON.stringify(initialMaxTokensPerModel) @@ -200,6 +206,7 @@ if (Object.keys(aiProviders).length < 1) { codeCompletionModel = undefined defaultModel = undefined + metadataModel = undefined } }) @@ -239,6 +246,10 @@ defaultModel && modelProviderMap[defaultModel] ? { model: defaultModel, provider: modelProviderMap[defaultModel] } : undefined + const metadata_model = + metadataModel && modelProviderMap[metadataModel] + ? { model: metadataModel, provider: modelProviderMap[metadataModel] } + : undefined const custom_prompts: Record = Object.entries(customPrompts) .filter(([_, prompt]) => prompt.trim().length > 0) .reduce((acc, [mode, prompt]) => ({ ...acc, [mode]: prompt }), {}) @@ -248,6 +259,7 @@ providers: aiProviders, code_completion_model, default_model, + metadata_model, custom_prompts: Object.keys(custom_prompts).length > 0 ? custom_prompts : undefined, max_tokens_per_model: Object.keys(maxTokensPerModel).length > 0 ? maxTokensPerModel : undefined @@ -258,6 +270,7 @@ function isSaveDisabled(): boolean { return ( !Object.values(aiProviders).every((p) => p.resource_path) || + (metadataModel != undefined && metadataModel.length === 0) || (codeCompletionModel != undefined && codeCompletionModel.length === 0) || (Object.keys(aiProviders).length > 0 && !defaultModel) ) @@ -397,6 +410,14 @@ codeCompletionModel = undefined } } + if (metadataModel) { + const currentMetadataModel = Object.values(aiProviders).find( + (p) => metadataModel && p.models.includes(metadataModel) + ) + if (!currentMetadataModel) { + metadataModel = undefined + } + } } }} /> @@ -478,6 +499,23 @@ {/key} + + {#key Object.keys(aiProviders).length} + {/if} diff --git a/frontend/src/lib/components/InstanceSettings.svelte b/frontend/src/lib/components/InstanceSettings.svelte index 7342f1d3fa..9581357e84 100644 --- a/frontend/src/lib/components/InstanceSettings.svelte +++ b/frontend/src/lib/components/InstanceSettings.svelte @@ -13,6 +13,7 @@ import { createEventDispatcher } from 'svelte' import { setLicense } from '$lib/enterpriseUtils' import AuthSettings from './AuthSettings.svelte' + import oauthConnectRegistry from '$oauth_connect_registry' import InstanceSetting from './InstanceSetting.svelte' import { writable, type Writable } from 'svelte/store' import { ExternalLink, Loader2 } from 'lucide-svelte' @@ -54,7 +55,9 @@ let initialValues: Record = $state({}) let baseUrlIsFallback = $state(false) - let snowflakeAccountIdentifier = $state('') + // Per-instance OAuth providers (Snowflake, ServiceNow, …): instance name + // keyed by provider, used to build their per-instance connect_config URLs. + let instanceInputs: Record = $state({}) let version: string = $state('') let loading = $state(true) @@ -147,12 +150,8 @@ $values = nvalues loading = false - // populate snowflake account identifier from db - const account_identifier = - oauths?.snowflake_oauth?.connect_config?.extra_params?.account_identifier - if (account_identifier) { - snowflakeAccountIdentifier = account_identifier - } + // populate per-instance OAuth provider inputs (snowflake, servicenow, …) from db + loadInstanceInputs(oauths) } export async function saveSettings() { @@ -162,13 +161,7 @@ } } - if ( - oauths?.snowflake_oauth && - oauths?.snowflake_oauth?.connect_config?.extra_params?.account_identifier !== - snowflakeAccountIdentifier - ) { - setupSnowflakeUrls() - } + setupTemplatedOauthUrls() // Remove empty or invalid entries for critical error channels $values.critical_error_channels = $values.critical_error_channels.filter((entry: any) => { @@ -283,19 +276,54 @@ } } - function setupSnowflakeUrls() { - // strip all whitespaces from account identifier - snowflakeAccountIdentifier = snowflakeAccountIdentifier.replace(/\s/g, '') + // Per-instance OAuth providers (Snowflake, ServiceNow, …) keyed by name -> + // their registry connect_config_template. Adding a new one needs only a + // registry entry — no code here. + const connectConfigTemplates: Record = Object.fromEntries( + Object.entries(oauthConnectRegistry) + .filter(([, cfg]) => cfg && typeof cfg === 'object' && 'connect_config_template' in cfg) + .map(([name, cfg]) => [name, (cfg as any).connect_config_template]) + ) - const connect_config = { - scopes: [], - auth_url: `https://${snowflakeAccountIdentifier}.snowflakecomputing.com/oauth/authorize`, - token_url: `https://${snowflakeAccountIdentifier}.snowflakecomputing.com/oauth/token-request`, - req_body_auth: false, - extra_params: { account_identifier: snowflakeAccountIdentifier }, - extra_params_callback: {} + function normalizeInstanceInput(tmpl: any, raw: string): string { + let v = (raw ?? '').replace(/\s/g, '') + if (tmpl.strip_suffix) { + // accept a full host/URL or a bare name -> reduce to the bare instance + v = v.replace(/^https?:\/\//, '').replace(/\/.*$/, '') + if (v.endsWith(tmpl.strip_suffix)) { + v = v.slice(0, -tmpl.strip_suffix.length) + } + } + return v + } + + // Build each per-instance provider's connect_config from the admin-entered + // instance name + its registry template (substituting {instance} into the + // URLs). Replaces the old per-provider setup functions. + function setupTemplatedOauthUrls() { + for (const [name, tmpl] of Object.entries(connectConfigTemplates)) { + if (!oauths?.[name]) continue + const key = tmpl.extra_params_key ?? 'instance' + const v = normalizeInstanceInput(tmpl, instanceInputs[name] ?? '') + instanceInputs[name] = v + if (oauths[name].connect_config?.extra_params?.[key] === v) continue + oauths[name].connect_config = { + scopes: [], + auth_url: tmpl.auth_url.replaceAll('{instance}', v), + token_url: tmpl.token_url.replaceAll('{instance}', v), + req_body_auth: tmpl.req_body_auth ?? false, + extra_params: { [key]: v }, + extra_params_callback: {} + } + } + } + + // Recover the instance-name inputs from a saved oauths config (for load/discard). + function loadInstanceInputs(savedOauths: Record) { + for (const [name, tmpl] of Object.entries(connectConfigTemplates)) { + const key = tmpl.extra_params_key ?? 'instance' + instanceInputs[name] = savedOauths?.[name]?.connect_config?.extra_params?.[key] ?? '' } - oauths['snowflake_oauth'].connect_config = connect_config } let sendingStats = $state(false) @@ -510,9 +538,7 @@ if (category === 'Auth/OAuth/SAML') { oauths = JSON.parse(JSON.stringify(initialOauths)) requirePreexistingUserForOauth = initialRequirePreexistingUserForOauth - const account_identifier = - initialOauths?.snowflake_oauth?.connect_config?.extra_params?.account_identifier - snowflakeAccountIdentifier = account_identifier ?? '' + loadInstanceInputs(initialOauths) } else if (category === 'Registries') { const v = initialValues['workspace_registries'] $values['workspace_registries'] = v !== undefined ? JSON.parse(JSON.stringify(v)) : undefined @@ -524,9 +550,7 @@ $values = JSON.parse(JSON.stringify(initialValues)) oauths = JSON.parse(JSON.stringify(initialOauths)) requirePreexistingUserForOauth = initialRequirePreexistingUserForOauth - const account_identifier = - initialOauths?.snowflake_oauth?.connect_config?.extra_params?.account_identifier - snowflakeAccountIdentifier = account_identifier ?? '' + loadInstanceInputs(initialOauths) if (yamlMode) { syncFormToYaml() } @@ -535,13 +559,7 @@ export async function saveCategorySettings(category: string) { // Category-specific pre-processing if (category === 'Auth/OAuth/SAML') { - if ( - oauths?.snowflake_oauth && - oauths?.snowflake_oauth?.connect_config?.extra_params?.account_identifier !== - snowflakeAccountIdentifier - ) { - setupSnowflakeUrls() - } + setupTemplatedOauthUrls() } if (category === 'Alerts' && $values?.critical_error_channels) { @@ -1116,7 +1134,7 @@ {:else if category == 'Auth/OAuth/SAML'} Date: Thu, 4 Jun 2026 21:01:57 +0200 Subject: [PATCH 48/61] add adobe acrobat sign icon (#9447) Adds AdobeAcrobatSignIcon.svelte and registers `adobe_acrobat_sign` in APP_TO_ICON_COMPONENT, for the Adobe Acrobat Sign hub integration (windmill-labs/windmill-integrations#143). Co-authored-by: Claude Opus 4.8 (1M context) --- .../icons/AdobeAcrobatSignIcon.svelte | 24 +++++++++++++++++++ frontend/src/lib/components/icons/index.ts | 2 ++ 2 files changed, 26 insertions(+) create mode 100644 frontend/src/lib/components/icons/AdobeAcrobatSignIcon.svelte diff --git a/frontend/src/lib/components/icons/AdobeAcrobatSignIcon.svelte b/frontend/src/lib/components/icons/AdobeAcrobatSignIcon.svelte new file mode 100644 index 0000000000..2f04246966 --- /dev/null +++ b/frontend/src/lib/components/icons/AdobeAcrobatSignIcon.svelte @@ -0,0 +1,24 @@ + + + + + + + diff --git a/frontend/src/lib/components/icons/index.ts b/frontend/src/lib/components/icons/index.ts index daadb3449b..aa159f3a96 100644 --- a/frontend/src/lib/components/icons/index.ts +++ b/frontend/src/lib/components/icons/index.ts @@ -27,6 +27,7 @@ import QRCodeIcon from './QRCodeIcon.svelte' import LinkedinIcon from './LinkedinIcon.svelte' import HubspotIcon from './HubspotIcon.svelte' import DatadogIcon from './DatadogIcon.svelte' +import AdobeAcrobatSignIcon from './AdobeAcrobatSignIcon.svelte' import StripeIcon from './StripeIcon.svelte' import TelegramIcon from './TelegramIcon.svelte' import FunkwhaleIcon from './FunkwhaleIcon.svelte' @@ -245,6 +246,7 @@ export const APP_TO_ICON_COMPONENT = { linkedin: LinkedinIcon, hubspot: HubspotIcon, datadog: DatadogIcon, + adobe_acrobat_sign: AdobeAcrobatSignIcon, stripe: StripeIcon, telegram: TelegramIcon, funkwhale: FunkwhaleIcon, From 00a96b82f35680e4b3947c4d57bdc63e5ea856d6 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Thu, 4 Jun 2026 21:02:51 +0200 Subject: [PATCH 49/61] add databricks icon (#9445) Adds DatabricksIcon.svelte (brand mark, #FF3621) and registers it under `databricks` in the shared APP_TO_ICON_COMPONENT map, so both the app and hub frontends pick it up for the new Databricks hub integration. Co-authored-by: Claude Opus 4.8 (1M context) Co-authored-by: Ruben Fiszel --- .../components/icons/DatabricksIcon.svelte | 21 +++++++++++++++++++ frontend/src/lib/components/icons/index.ts | 2 ++ 2 files changed, 23 insertions(+) create mode 100644 frontend/src/lib/components/icons/DatabricksIcon.svelte diff --git a/frontend/src/lib/components/icons/DatabricksIcon.svelte b/frontend/src/lib/components/icons/DatabricksIcon.svelte new file mode 100644 index 0000000000..e767337442 --- /dev/null +++ b/frontend/src/lib/components/icons/DatabricksIcon.svelte @@ -0,0 +1,21 @@ + + + + + diff --git a/frontend/src/lib/components/icons/index.ts b/frontend/src/lib/components/icons/index.ts index aa159f3a96..56d77c810c 100644 --- a/frontend/src/lib/components/icons/index.ts +++ b/frontend/src/lib/components/icons/index.ts @@ -27,6 +27,7 @@ import QRCodeIcon from './QRCodeIcon.svelte' import LinkedinIcon from './LinkedinIcon.svelte' import HubspotIcon from './HubspotIcon.svelte' import DatadogIcon from './DatadogIcon.svelte' +import DatabricksIcon from './DatabricksIcon.svelte' import AdobeAcrobatSignIcon from './AdobeAcrobatSignIcon.svelte' import StripeIcon from './StripeIcon.svelte' import TelegramIcon from './TelegramIcon.svelte' @@ -246,6 +247,7 @@ export const APP_TO_ICON_COMPONENT = { linkedin: LinkedinIcon, hubspot: HubspotIcon, datadog: DatadogIcon, + databricks: DatabricksIcon, adobe_acrobat_sign: AdobeAcrobatSignIcon, stripe: StripeIcon, telegram: TelegramIcon, From fee23a51859d84e843a789958db8eae6b771bacc Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 5 Jun 2026 01:00:12 +0000 Subject: [PATCH 50/61] threat_model v0 --- backend/THREAT_MODEL.md | 172 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 172 insertions(+) create mode 100644 backend/THREAT_MODEL.md diff --git a/backend/THREAT_MODEL.md b/backend/THREAT_MODEL.md new file mode 100644 index 0000000000..5a891bda77 --- /dev/null +++ b/backend/THREAT_MODEL.md @@ -0,0 +1,172 @@ +# Threat Model: Windmill Backend + +## 1. System context + +Windmill is an open-source (AGPLv3) developer platform for internal tools, +workflows, background jobs, API integrations, and UIs — a self-hostable +alternative to Retool / Pipedream / Airplane. The backend is a Rust workspace +(~60 crates: `windmill-api`, `windmill-worker`, `windmill-queue`, +`windmill-common`, a family of `windmill-trigger-*` crates, `windmill-mcp`, +`windmill-sandbox`, etc.) fronting a PostgreSQL database. A Svelte 5 frontend +(not in scope here, but referenced where stored-XSS threats originate) is +served by the same instance. The product ships in a Community Edition (CE, +public Docker images) and an Enterprise Edition (EE, `*_ee.rs` files gated by +`enterprise`/`private`/`license` cargo features). + +The defining characteristic for threat modeling is that **Windmill executes +arbitrary user-supplied code** (Python, TypeScript via Bun/Deno, Go, Bash, +SQL, GraphQL, PowerShell, Rust, …) on its workers, and **stores the +credentials to every system its users connect to** (databases, cloud +accounts, SaaS APIs, OAuth tokens). It is therefore simultaneously an +arbitrary-code-execution engine and a credential vault — compromising one +instance can pivot into an organization's entire connected estate. Crucially, +the owner confirms `nsjail` is **off by default everywhere** (`ENABLE_NSJAIL` +is opt-in) and network isolation (`clone_newnet`) is separately gated: the +*only* job isolation present in a default install is PID-namespace `unshare`. +Filesystem and outbound-network isolation are therefore absent unless an +operator deliberately enables them, which makes "weak-by-default isolation" a +more accurate frame than "sandbox escape" for typical deployments. Cross-tenant +separation is enforced in software via workspace IDs, token scopes, folder +ACLs, and Postgres row-level security; on the managed offering, sensitive +customers can opt into dedicated DB / worker / namespace infrastructure, but +the shared tier relies entirely on that software boundary. Administrators are +strongly encouraged to use nsjail sandboxing and are reminded that if they don't, +their security model is that they trust their developers that write code ran on windmill +to not do anything TOO malicious on the workers. When the default +database secret backend is used, only per-workspace secret *variables* are +encrypted at rest — instance-level `global_settings` (OAuth client secrets, +SMTP, object-store keys, license) are stored plaintext, so a database read +yields the instance-wide credential set. Internet-facing instances are +typically exposed directly with no built-in rate limiting or WAF. + +It is deployed self-hosted (Docker Compose, Kubernetes/Helm, bare metal), on +cloud providers, and as a Windmill-Labs-managed multi-tenant service. The API +server is internet-facing in most deployments; workers pull jobs from the +Postgres queue. The large public attack surface (a sprawling authenticated +HTTP API, unauthenticated public-app and webhook/trigger endpoints, outbound +HTTP from user code and proxies) combined with the high-value assets makes +authorization-enforcement bugs, SSRF, SQL injection, and sandbox escape the +dominant risk categories — a pattern strongly confirmed by the project's +published advisory history (73 GHSA advisories, several rated 9.9 critical). + +## 2. Assets + +| asset | description | sensitivity | +|---|---|---| +| Workspace encryption keys | Per-workspace key (`workspace_key`) used to encrypt secret variables (MagicCrypt256); decrypts all secrets in the workspace | critical | +| Secret variables | User secrets stored encrypted in `variable` (is_secret) | critical | +| Resource credentials | DB passwords, cloud creds, API keys, connection strings in `resource` JSONB | critical | +| OAuth / external-account tokens | Refresh/access tokens in `account`, MCP OAuth tables | critical | +| User password hashes | Argon2 hashes in `password` table | critical | +| API tokens & session cookies | Bearer tokens / cookies in `token`; superadmin & scoped tokens | critical | +| Instance global settings | License key, JWT secret, SUPERADMIN_SECRET, SMTP, object-store + secret-backend (Vault/KMS/SM) creds in `global_settings` | critical | +| Worker host & process integrity | The host that runs untrusted user code | critical | +| Cross-tenant / cross-workspace isolation | The software boundary separating workspaces, folders, and tenants | critical | +| Downstream connected systems | Windmill is a credential vault: stored creds reach external DBs, cloud accounts, SaaS | critical | +| Script / flow / app source | Customer IP & business logic in `script`, `flow`, `app`, `raw_app` | high | +| Job arguments, results & logs | `queue`/`completed_job` args+result, `job_logs`; routinely contain secrets | high | +| Object store / S3 data | Files uploaded/produced by jobs | high | +| Audit logs | `audit`/`audit_partitioned` action trail | high | +| Service availability | API server + worker fleet uptime | high | +| PII | User emails, group membership | medium | + +## 3. Entry points & trust boundaries + +| entry_point | description | trust_boundary | reachable_assets | +|---|---|---|---| +| EP1 Authenticated job-execution API | `jobs/run/preview`, `run/h/{hash}`, `run_flow/run_script` — runs user code on workers | authenticated user → arbitrary code on worker | Worker host, downstream systems, isolation, job args/results/logs | +| EP2 Unauthenticated public endpoints | `apps_u/*`, `jobs_u/getupdate*`, `scripts_u`, `settings_u`, `resources_u` (`public_app_layer.rs`) | unauth HTTP → app logic & job data | Job results, scripts, secrets, PII | +| EP3 HTTP-trigger & webhook ingestion | `/api/r/*`, GCP/Azure push, Slack callback, `capture_u/*` | untrusted webhook → job queue | Job execution integrity, worker host | +| EP4 Message-queue / native triggers | kafka, postgres, mqtt, websocket, nats, sqs, email triggers | external broker/message → job queue | Job execution integrity, availability | +| EP5 HTTP API authorization layer | Token/scope/RLS/folder-ACL enforcement across all workspaced routes (`windmill-api-auth`) | scoped token / low-priv user → other users' & workspaces' data | Scripts, job data, secrets, isolation | +| EP6 AI proxy & MCP endpoints | `ai/proxy/*`, `mcp` — resolve `$var:`/resources, proxy to LLM APIs, `X-Resource-Path` | authenticated user → outbound HTTP + secret resolution | Secrets, resource creds, internal network, downstream | +| EP7 Outbound HTTP from executors/resources | GraphQL/HTTP/Postgres executors, webhook delivery, `test_object_storage_config`, git clone, npm tarball fetch | user-controlled URL → server-side request | Cloud metadata, internal network, downstream creds | +| EP8 SQL query builders & contextual-var substitution | App DB query builder (`whereClause`/`tags`), Postgres-trigger `where_clause`, `%%WM_*%%` interpolation, `WM_INTERNAL_DB` | user input → raw SQL | Database, connected DBs | +| EP9 Worker sandbox | nsjail / unshare / dind / rootless podman isolating user code | user code → host & cross-tenant filesystem/network | Worker host, isolation, downstream | +| EP10 Worker code generation / wrappers | Entrypoint override, env-var names, workspace env interpolated into generated wrapper code | user-controlled identifier → executable code | Worker host, isolation | +| EP11 OAuth / OIDC / SAML / MCP-OAuth / logout | Login callbacks, MCP OAuth client registration, logout `rd` redirect | untrusted IdP / redirect input → session | Session tokens, accounts | +| EP12 Stored-content rendering | App builder HTML component, markdown, S3 download response headers | stored user content → admin browser (same origin) | Admin session, account takeover | +| EP13 Log/file reading & export endpoints | `service_logs`, `jobs_u/getupdate` log file read (symlinks), workspace/tarball export | authed/unauth request → arbitrary file or admin-only config | Arbitrary files, global settings | +| EP14 Secret-value & resource-value caches | In-memory caches in `windmill-store` keyed (historically un-keyed) by path | cache lookup crossing identity/folder boundary | Secret variables, resource creds | +| EP15 Deployment & runtime config | docker-compose defaults: dind, debugger (`REQUIRE_SIGNED_DEBUG_REQUESTS=false`), CORS `Any`, default admin/`changeme`, exposed Postgres, `SUPERADMIN_SECRET`, `ENABLE_NSJAIL=false`, privileged containers | operator/infra default → full instance | All assets | +| EP16 Supply chain | Cached hub scripts, GitHub workflow actions, vendored deps, Docker base image | build/update-time input → host & build integrity | Worker host, build integrity | +| EP17 Token lifecycle | Token create/rescope/refresh, script-issued JWTs | scoped caller → broader privilege | Tokens, accounts, isolation | + +## 4. Threats + +| id | threat | actor | surface | asset | impact | likelihood | status | controls | evidence | +|---|---|---|---|---|---|---|---|---|---| +| T1 | SQL injection in app/internal query builders and trigger clauses compromises the metadata DB and connected databases | remote_auth | EP8 | Database, downstream connected systems | critical | almost_certain | partially_mitigated | sqlx parameterized queries elsewhere; query-builder safety reviews | GHSA-225c-j3xq-g6x6, GHSA-78p7-jc72-gv66, GHSA-hvc7-f67h-jx3g, GHSA-wrrg-f89m-f84q, GHSA-79vf-3qwm-2w64, GHSA-55p6-fxj4-v983, GHSA-5g4v-49rj-r52r, GHSA-x6cq-7xr8-53x3, 2cf4bb180b | +| T2 | Server-side request forgery via proxies/executors reaches cloud metadata, internal network, and downstream credentials | remote_auth | EP6, EP7 | Cloud metadata, internal network, downstream connected systems, resource creds | critical | almost_certain | partially_mitigated | SSRF URL validation + redirect-following disabled added piecemeal; outbound network isolation (`clone_newnet`) is opt-in and off by default | GHSA-3ggp-h37f-5qfw, GHSA-98qq-g8rh-xhff, GHSA-hfw8-27mx-63jm, GHSA-3r59-qvvc-774j, GHSA-4pj9-w5jc-g8w7, GHSA-8hh3-jf25-78j5, GHSA-3pjm-4w7f-3r2w, GHSA-f44c-x9hq-h68r, GHSA-j4h4-f8fj-3m3c, 4b06881918, 96a8eb63d4, dbd3942ef3 | +| T3 | Broken authorization / IDOR lets a scoped token or low-privilege member read scripts, job data, and secrets across folders and workspaces | remote_auth | EP5, EP2, EP1 | Scripts, job data, secrets, isolation | critical | almost_certain | partially_mitigated | RLS, token scopes, folder ACLs, view-token HMAC (added incrementally); on managed, sensitive tenants can opt into dedicated DB/worker/namespace, but the shared tier IS the software boundary | GHSA-qfg7-x243-5hg4, GHSA-8x8x-88qc-qp4r, GHSA-2ppx-66jv-wpw5, GHSA-x3x7-g97v-mp59, GHSA-j276-g4h8-g6h5, GHSA-8mv7-hmrg-96xv, GHSA-x2wf-f962-7frq, GHSA-qc7c-gcw6-h4xp, GHSA-vxc5-w28p-m9xw, GHSA-2g34-wfvr-5qqj, GHSA-w7p6-wpxm-pp66, 7edf3f0212, 89a7a37776, ab11c7747a, 664edcdfb7 | +| T4 | Remote code execution by injecting attacker-controlled identifiers into generated worker wrappers | remote_auth | EP10 | Worker host, isolation, downstream | critical | likely | partially_mitigated | entrypoint/env-var-name validation added | GHSA-wxjq-w5pj-jqhx, GHSA-5f5q-2vg2-r2x4, GHSA-8q8j-mm3g-5c2q (CVE-2026-33881), bf93657fee, bd05bcadde, 22ec4da5f0 | +| T5 | Worker compromise & cross-tenant access via weak-by-default isolation (nsjail off by default → user code runs with only PID-ns `unshare`); sandbox escape where nsjail/dind/podman is enabled | remote_auth | EP9, EP15 | Worker host, isolation, downstream | critical | likely | unmitigated | nsjail off by default everywhere (`DISABLE_NSJAIL=true`); shipped compose gives PID-ns `unshare` only (`FAVOR_UNSHARE_PID=true`), bare installs get no isolation. Where nsjail enabled: read-only remounts, jail-tmp refusal, podman socket gating | GHSA-6qr8-xhg4-453q, GHSA-3vpp-vf62-wqp6, f8467f38c8, df5aec0f5d, f1b6746e0e | +| T6 | Disclosure of secrets, resource credentials, and workspace encryption keys across the authorization boundary (AI proxy, MCP, caches, export); database read additionally yields plaintext instance-level `global_settings` secrets | remote_auth | EP6, EP14, EP13 | Secret variables, encryption keys, resource creds, global settings | critical | likely | partially_mitigated | RLS on `$var:`, cache scoping by caller, admin checks on export; per-workspace secret *variables* encrypted at rest, but `global_settings` is plaintext under the default DB secret backend | GHSA-jwg4-v3cj-rvfm, GHSA-8m2p-2crh-9h3w, GHSA-6635-6fch-v8px, GHSA-437f-725p-7w84, GHSA-f27g-j463-q85w (CVE-2026-26964), GHSA-j679-v6vj-jfxc, GHSA-6vrr-fq33-qpfp, 0ba128afe7, 7836a4e733, ff8e39c69b | +| T7 | Full instance compromise from insecure deployment defaults (dind control, default admin/`changeme`, exposed Postgres, publicly readable SUPERADMIN_SECRET) | remote_unauth | EP15 | All assets | critical | likely | partially_mitigated | first-time-setup warning on default admin; docs recommend hardening | GHSA-3vpp-vf62-wqp6, GHSA-24fr-44f8-fqwg (CVE-2026-29059), GHSA-6q36-5p3h-766j | +| T8 | Unauthenticated RCE via the Debugger WebSocket in the default `windmill_extra` configuration | remote_unauth | EP15 | Worker host, all assets | critical | possible | unmitigated | `REQUIRE_SIGNED_DEBUG_REQUESTS` exists but defaults to false | GHSA-725h-99vx-9xr4 | +| T9 | Supply-chain compromise via cached hub scripts, GitHub workflow command injection, or vulnerable base-image deps | supply_chain | EP16 | Worker host, build integrity | critical | possible | partially_mitigated | hub-script re-pin to patched versions; HUB_BASE_URL override | GHSA-w2m9-q5f7-3gpq, edf340c4d4, GHSA-8rq7-w7g6-8wvr, GHSA-vch9-39v5-4wg7 (CVE-2024-37371) | +| T10 | Unauthenticated disclosure of job results, args, logs, and admin config via missing-authz public endpoints | remote_unauth | EP2, EP13 | Job results/args/logs, global settings, scripts | high | likely | partially_mitigated | anonymous-job checks, log-endpoint authz hardening | GHSA-qfg7-x243-5hg4, GHSA-v448-fmm4-52fp, 108a88a180, bb90f4ce83 | +| T11 | Stored XSS leading to admin/account takeover via app HTML component, markdown, or S3 download content-type | remote_auth | EP12 | Admin session, accounts | high | likely | partially_mitigated | DOMPurify markdown sanitization, `X-Content-Type-Options: nosniff` + CSP sandbox on downloads | GHSA-9c5c-hh3c-r9mc, GHSA-qxj7-hpx3-r892, GHSA-cf2x-rg8c-v63v, bb78b1c06d, 625b67dff0 | +| T12 | Webhook authentication bypass / signature replay forges trigger invocations and approvals | remote_unauth | EP3 | Job execution integrity, approvals | high | likely | partially_mitigated | HMAC verification on some triggers; signing-oracle fix | GHSA-jw8c-h45c-xpjw, GHSA-hh9x-rcf8-xjr2, GHSA-q9g3-q6fj-hc2x, GHSA-8jc4-wj2p-2vmp, ab2a15b2a8 | +| T13 | Path traversal / arbitrary file read via log-reading and MCP path endpoints (incl. symlink following) | remote_auth | EP13 | Arbitrary files on server, global settings | high | likely | partially_mitigated | traversal checks + no-symlink-follow added | GHSA-4hrf-mgvv-xp9x, bb90f4ce83, df451aa64f, ad5ec293b5, 5f2d3e6812 | +| T14 | Privilege escalation via token rescope/refresh, script-issued JWTs, or operator-permission gaps | remote_auth | EP17, EP5 | Tokens, isolation, accounts | high | likely | partially_mitigated | monotonic-privilege enforcement on token lifecycle; SECURITY DEFINER triggers | GHSA-p62p-67xp-v775, GHSA-vv9w-wx3c-q3x2, 2ddf93de96, 865ab70c89, 33fb08cf3d | +| T15 | Credential leakage via worker `/proc` environment and unmasked secrets in job logs | remote_auth | EP9, EP1 | DB creds, secrets, downstream | high | likely | partially_mitigated | Aho-Corasick secret masking in logs | GHSA-pmp9-9924-f9cx, 0885d8c986 | +| T16 | Denial of service via resource exhaustion: unbounded uploads, runaway jobs, queue flooding, or trigger-message storms | remote_auth | EP1, EP3, EP4 | Service availability, worker fleet | high | likely | risk_accepted | Per-job rlimits/timeouts exist; instance-wide DoS by an authenticated tenant is largely accepted on shared self-host (operator's job to add global quotas). Hard requirement only for managed multi-tenant | | +| T17 | Account/credential theft via unauthenticated MCP-OAuth client registration and open redirect on logout | remote_unauth | EP11 | Accounts, session tokens | high | possible | partially_mitigated | redirect-URI handling / registration hardening | GHSA-q9xg-f2v2-695g, GHSA-53xj-pvqf-wpm9, GHSA-rr8j-ffc4-pf7h, GHSA-6c5w-777m-8rv5 | +| T18 | Account takeover via missing rate limiting / brute force on auth endpoints | remote_unauth | EP11 | Accounts | medium | likely | unmitigated | none built-in; owner confirms instances are typically exposed directly with no app-level rate limiting or WAF | GHSA-cmv6-m7wc-c87p | +| T19 | Enterprise license bypass and account impersonation | remote_auth | EP5 | Global settings, accounts | medium | possible | unmitigated | license validation gated by `license` feature | GHSA-48j5-p323-4mpx, GHSA-pv35-65rq-w29h, GHSA-2qx7-634r-qj6r | +| T20 | Trigger spoofing: an actor with broker/queue access injects messages that execute jobs without app-level auth | adjacent_network | EP4 | Job execution integrity, downstream | medium | possible | risk_accepted | Owner confirms trust is delegated to broker ACLs by design; no app-level message authenticity check. Anyone able to publish to a subscribed topic/queue can cause job execution | | +| T21 | Data-in-transit interception/tampering from TLS-disabled defaults (DB `sslmode=disable`, HTTP-only Caddy) | adjacent_network | EP15 | DB creds, secrets, session tokens | medium | possible | unmitigated | docs recommend TLS; not default | | +| T22 | Repudiation / incident blind spots from gaps in audit coverage of sensitive actions | remote_auth | EP5 | Audit logs | medium | possible | partially_mitigated | `windmill-audit` records many actions | | + +## 5. Deprioritized + +| threat | reason | +|---|---| +| Physical access to the host / cold-boot key extraction | Out of scope; deployment-environment responsibility, not addressable in this codebase | +| Memory-safety RCE in the Rust backend itself | Rust's safety model makes this rare; no evidence in history. Note: `unsafe` FFI (duckdb) is a narrow exception folded into supply-chain/T9 | +| Client-side-only nuisance bugs (CSS, layout) with no security impact | No asset compromised | +| Insider with legitimate superadmin / DB-root access | Trusted role; mitigations are operational (least privilege, audit), not technical controls in scope | +| Spoofing of a fully-trusted upstream IdP that has itself been compromised | Out of model; Windmill trusts the configured IdP by design | +| Instance-wide DoS by an authenticated tenant on shared self-host (T16) | Risk accepted (owner): per-job rlimits/timeouts are in place; global concurrency/queue quotas are the operator's responsibility on self-host. Remains a hard requirement for the managed multi-tenant fleet | +| Job execution triggered by an actor with legitimate broker/queue publish access (T20) | Risk accepted (owner): trigger authenticity is delegated to broker ACLs by design; consuming from a configured source and acting on its messages is the intended behavior | + +## 6. Open questions + +Facts that drove the score changes above. Two were confirmed in code during +the interview (`[Code-verified]`); the rest remain `[Owner-states]` pending a +check. + +- [Code-verified] nsjail is off by default in every configuration: `DISABLE_NSJAIL` defaults to `true` (`windmill-worker/src/worker.rs:346`), and `is_sandboxing_enabled()` requires `DISABLE_NSJAIL=false` or the `job_isolation` global setting = `nsjail_sandboxing` (`worker.rs:890`). PID-ns `unshare` is also off at the code level (`is_unshare_enabled()`, `worker.rs:903`); the shipped `docker-compose.yml` sets `FAVOR_UNSHARE_PID=true` (line 91), so the official compose gives PID-ns unshare only, nsjail off — a bare install gets no isolation at all. No separate `clone_newnet` flag exists; network isolation is an nsjail feature, so outbound network from user code is unrestricted by default. Affects: T2 controls/likelihood, T5 status (unmitigated), T8. +- [Code-verified] `global_settings` is plaintext at rest under the default DB backend: `set_value_in_global_settings` stores the raw JSON value with no encryption (`windmill-common/src/global_settings.rs:259`); the encrypting secret backend (`secret_backend/database.rs:66`) only encrypts per-workspace `variable` rows with `is_secret=true`. Instance-level SMTP/OAuth/AI/object-store secrets are therefore plaintext. Affects: T6 impact/controls, T7. +- [Owner-states] Internet-facing instances are typically exposed directly with no built-in rate limiting / WAF. Affects: T16, T18 likelihood. Verify by: confirm absence of a rate-limit layer in `windmill-api/src/lib.rs` middleware stack. +- [Owner-states] Managed offering provides an optional dedicated DB/worker/namespace tier for sensitive tenants; the shared tier relies solely on the software authz boundary. Affects: T3 controls. Verify by: deployment topology (not in this repo) — out-of-tree. +- [Owner-states] Per-job rlimits/timeouts exist; instance-wide DoS by an authed tenant is risk-accepted on shared self-host. Affects: T16 status. Verify by: locate the rlimit/timeout enforcement in the worker execution path and confirm there is no global queue/concurrency cap. +- [Owner-states] Message-queue trigger authenticity is delegated to broker ACLs only. Affects: T20 status. Verify by: review `windmill-trigger-{kafka,sqs,nats,mqtt,postgres}` consume paths for any payload authentication. + +## 7. Provenance + +- mode: bootstrap-then-interview +- date: 2026-06-05 +- target: /home/rfiszel/windmill/backend @ 819ba5e150 +- inputs: git-log mined + GitHub security advisories (gh api, 73 advisories) + CHANGELOG; seed: THREAT_MODEL.md (bootstrap pass) +- owner: Ruben Fiszel (Windmill core dev) + +## 8. Recommended mitigations + +| mitigation | threat_ids | closes_class | effort | +|---|---|---|---| +| Centralize a single audited query-builder that forbids string-interpolated SQL; ban `format!`-built queries via lint/CI | T1 | yes | M | +| Route all outbound requests through one SSRF-guarded HTTP client (allowlist/denylist of private+metadata ranges, redirects disabled, re-validated per hop) | T2 | yes | M | +| Enforce authorization centrally in middleware (scope + RLS + folder ACL) with deny-by-default and a per-route coverage test, instead of per-handler checks | T3, T10, T14, T22 | yes | L | +| Treat all user-supplied identifiers as data: pass via argv/env/structured params, never splice into generated wrapper source; validate against strict allowlists at the boundary | T4 | yes | M | +| Make `nsjail` + network-namespace isolation default-on / fail-closed (flip `ENABLE_NSJAIL` and `clone_newnet` defaults) and remove privileged/dind defaults from shipped compose; default-deny debugger | T2, T5, T7, T8 | partial | L | +| Encrypt `global_settings` at rest under the workspace/instance key even on the default DB secret backend, so a DB read no longer yields plaintext instance-wide credentials | T6, T7 | partial | M | +| Ship hardened defaults: random per-install secrets, no default admin password, Postgres not exposed, CORS locked to configured origin, TLS-on | T7, T18, T21 | partial | M | +| Resolve secrets/resources only with the caller's identity and scope every cache entry by (caller, scope); apply uniformly to AI proxy, MCP, and exports | T6 | yes | M | +| Output-encode/sanitize all stored content at render and force `nosniff` + restrictive CSP on every user-content response | T11 | yes | M | +| Verify webhook authenticity uniformly (constant-time HMAC + timestamp/nonce anti-replay) in a shared trigger-auth helper | T12 | yes | S | +| Canonicalize + confine all file-path inputs to a base dir and never follow symlinks in log/file readers | T13 | yes | S | +| Mask secrets at the log sink and keep secrets out of worker process env (`/proc`) — pass via files/pipes scrubbed after use | T15 | partial | M | +| Add global rate limiting and per-tenant resource/queue quotas at the edge | T16, T18 | partial | M | +| Pin and integrity-verify hub scripts and CI actions; SBOM + automated base-image CVE scanning in release | T9 | partial | M | From fb175e1c9d24533caab1c771e97d817b052ee3d0 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Fri, 5 Jun 2026 10:07:01 +0200 Subject: [PATCH 51/61] fix ee repo ref dynamic oauth urls (#9451) * ee repo ref * fix(ee-ref): pin to EE commit that includes read_only create_session_token fix The previous pin (f7a83d9) carried only the connect_config_template change and dropped Ruben's read_only=false fix (EE 3742e06). CE #9371 made create_session_token require 6 args, so the EE overlay fails check_ee_full with an arity error without it. Bump the pin to 9be38de, which includes both fixes. Co-Authored-By: Claude Opus 4.8 (1M context) * chore: update ee-repo-ref to fb106b89cdf4088b004dac6062adb029f3923887 This commit updates the EE repository reference after PR #603 was merged in windmill-ee-private. Previous ee-repo-ref: 9be38def879f702cd0b134d9e71bbb17fbb9cfa4 New ee-repo-ref: fb106b89cdf4088b004dac6062adb029f3923887 Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Opus 4.8 (1M context) Co-authored-by: windmill-internal-app[bot] --- backend/ee-repo-ref.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 15e5b90000..73eaef904f 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -servicenow-oauth +fb106b89cdf4088b004dac6062adb029f3923887 From 1727271e197b34026efeaf1b6561bb404a440baa Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 5 Jun 2026 10:35:51 +0200 Subject: [PATCH 52/61] feat: sandboxed daemonless container runtime via '# sandbox ' (#9453) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add sandboxed docker v2 runtime via '# docker ' Run a container image as a subprogram of the job's own nsjail sandbox: extract the image rootfs with podman (rootless) and run it chrooted inside the job's nsjail, so the container inherits the job's confinement and is safe under nsjail / for untrusted code. Selected by '# docker '; a bare '# docker' keeps the v1 (dind) path untouched. Co-Authored-By: Claude Opus 4.8 (1M context) * feat: default to daemonless docker (drop dind from compose, allow docker on cloud) docker-compose no longer ships the dind sidecar (v2 is daemonless: podman + nsjail in the worker); removed the dind service, DOCKER_HOST env, depends_on and volume. Removed the language-picker guard that blocked Docker scripts on the multi-tenant platform, now that v2 makes docker safe to run sandboxed. Co-Authored-By: Claude Opus 4.8 (1M context) * feat: select sandboxed container via # sandbox ; add pull policy + size guards - Surface moved from '# docker ' to '# sandbox ' (groups under the sandbox annotation; '# docker' stays v1-only, '# sandbox' stays nsjail-bash). - SANDBOX_IMAGE_PULL_POLICY (default 'newer') so moving tags don't go stale. - SANDBOX_IMAGE_MAX_SIZE_MB rejects oversized images before extraction. - SANDBOX_IMAGE_CACHE_MAX_MB best-effort LRU eviction of podman's image store. Co-Authored-By: Claude Opus 4.8 (1M context) * feat(sandbox): support # volume, honor nsjail tmp instance settings, v2 docker template - Thread shared_mount into the sandbox container nsjail config so '# volume' mounts (and the same-worker /tmp/shared folder) apply inside the container. - Use resolve_nsjail_tmp_mount_block for the container's /tmp so it honors the same nsjail_tmp_backing / nsjail_tmpfs_size_mb instance settings as other nsjail jobs. - docker-compose comment + the editor's Docker template now use '# sandbox '. Co-Authored-By: Claude Opus 4.8 (1M context) * feat(sandbox): make image size/cache/pull-policy UI instance settings Convert SANDBOX_IMAGE_* from worker env vars to DB-backed instance settings (sandbox_image_max_size_mb, sandbox_image_cache_max_mb, sandbox_image_pull_policy), hot-reloaded via the same mechanism as nsjail_tmpfs_size_mb and configurable in #superadmin-settings. No worker restart needed. Co-Authored-By: Claude Opus 4.8 (1M context) * feat(sandbox): windmill-managed registry — default registry + private auth Two new instance settings: - sandbox_image_default_registry: prepended to unqualified image refs (alpine -> /alpine); fully-qualified refs untouched. - sandbox_registry_auth: docker/podman auth.json blob written to a per-job authfile (0600, removed with the job) and passed to podman --authfile for private registries. Both hot-reloaded and configurable in #superadmin-settings. Co-Authored-By: Claude Opus 4.8 (1M context) * fix(sandbox): protobuf-safe proto_str escaper, atomic 0600 authfile, registry tests Addresses local-review P2s: proto_str now emits valid protobuf octal escapes for control/non-ASCII bytes (not Rust \u{..} that nsjail would reject); the registry authfile is created 0600 atomically (no world-readable window); add a registry_qualified table test + a non-ASCII proto_str case. Co-Authored-By: Claude Opus 4.8 (1M context) * fix(sandbox): P0 — deliver image env via nsjail envar:, never the launcher process env CI review (P0): the image's OCI Env (attacker-controlled keys+values) was applied to the nsjail launcher process via .envs(), so a hostile image could set LD_PRELOAD/ LD_LIBRARY_PATH/LD_AUDIT on nsjail itself and execute code as the worker outside the jail. Now the image env is rendered as proto-escaped 'envar:' directives (child-only) and nsjail's process env carries only windmill-trusted keys (reserved vars + proxy). Also: warn instead of silently bypassing the size guard on inspect failure; reset the eviction guard via a Drop guard (no stuck flag on panic/early-return). +render_envars test. Co-Authored-By: Claude Opus 4.8 (1M context) * fix(sandbox): P0 symlink-write escape via rootfs script; P1 redact registry-auth logging CI review: - P0 (Codex): the body was written into the image-controlled rootfs as .windmill_docker_main.sh via write_file (follows symlinks) — a hostile image could plant that path as a symlink to a host file and capture the worker's write before nsjail starts. Now the body is passed straight to 'sh -c sh '; no file is written into the rootfs at all. - P1 (Codex): sandbox_registry_auth flowed through the generic setting loader which logs the value (raw auth.json credentials). Replaced with a secret-aware reload that loads directly and logs only a redacted 'configured=' message. Co-Authored-By: Claude Opus 4.8 (1M context) * fix(sandbox): redact sandbox_registry_auth in instance-settings write log too The settings API also logs 'Set global setting to ' via format_setting_value; add sandbox_registry_auth to SENSITIVE_SETTINGS so the credential is redacted there as well as on reload. * fix(sandbox): don't silently disable cache eviction on podman images parse error Re-review (cubic/Claude P2): serde_json::from_slice(...).unwrap_or_default() meant any parse hiccup (e.g. podman omitting Size/Created via omitempty for a zero value, or schema drift) silently degraded to an empty Vec and disabled eviction with no log. Now Size/Created are #[serde(default)] (a missing omitempty key -> 0, not a whole-array parse failure) and a real parse error warns + breaks instead of being swallowed. --------- Co-authored-by: Claude Opus 4.8 (1M context) --- backend/src/main.rs | 33 +- backend/src/monitor.rs | 75 +- .../windmill-common/src/global_settings.rs | 5 + .../windmill-common/src/instance_config.rs | 1 + backend/windmill-common/src/worker.rs | 59 ++ .../nsjail/run.docker.config.proto | 103 +++ backend/windmill-worker/src/bash_executor.rs | 36 +- backend/windmill-worker/src/common.rs | 10 + backend/windmill-worker/src/docker_v2.rs | 683 ++++++++++++++++++ backend/windmill-worker/src/lib.rs | 1 + backend/windmill-worker/src/worker.rs | 21 + docker-compose.yml | 35 +- docs/docker-v2-runtime.md | 106 +++ .../src/lib/components/ScriptBuilder.svelte | 15 - .../flows/content/FlowInputs.svelte | 19 - .../flows/content/FlowInputsQuick.svelte | 19 - .../src/lib/components/instanceSettings.ts | 54 ++ frontend/src/lib/script_helpers.ts | 21 +- 18 files changed, 1181 insertions(+), 115 deletions(-) create mode 100644 backend/windmill-worker/nsjail/run.docker.config.proto create mode 100644 backend/windmill-worker/src/docker_v2.rs create mode 100644 docs/docker-v2-runtime.md diff --git a/backend/src/main.rs b/backend/src/main.rs index e5daf3201b..b4d3cef4f9 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -57,11 +57,14 @@ use windmill_common::{ PREVIEW_TAGS_OVERRIDE_SETTING, REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RESTART_COORDINATION_SETTING, RETENTION_PERIOD_SECS_SETTING, RUBY_REPOS_SETTING, SAML_METADATA_SETTING, - SCIM_TOKEN_SETTING, SMTP_SETTING, STORE_AUDIT_LOGS_S3_SETTING, TEAMS_SETTING, - TIMEOUT_WAIT_RESULT_SETTING, UV_EXCLUDE_NEWER_SETTING, UV_INDEX_STRATEGY_SETTING, - UV_PYTHON_INSTALL_MIRROR_SETTING, WORKSPACE_FAIRNESS_DURATION_SECS_SETTING, - WORKSPACE_FAIRNESS_ENABLED_SETTING, WORKSPACE_FAIRNESS_MAX_PERCENT_SETTING, - WORKSPACE_FAIRNESS_MIN_TOTAL_SETTING, WORKSPACE_REGISTRIES_SETTING, + SANDBOX_IMAGE_CACHE_MAX_MB_SETTING, SANDBOX_IMAGE_DEFAULT_REGISTRY_SETTING, + SANDBOX_IMAGE_MAX_SIZE_MB_SETTING, SANDBOX_IMAGE_PULL_POLICY_SETTING, + SANDBOX_REGISTRY_AUTH_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING, + STORE_AUDIT_LOGS_S3_SETTING, TEAMS_SETTING, TIMEOUT_WAIT_RESULT_SETTING, + UV_EXCLUDE_NEWER_SETTING, UV_INDEX_STRATEGY_SETTING, UV_PYTHON_INSTALL_MIRROR_SETTING, + WORKSPACE_FAIRNESS_DURATION_SECS_SETTING, WORKSPACE_FAIRNESS_ENABLED_SETTING, + WORKSPACE_FAIRNESS_MAX_PERCENT_SETTING, WORKSPACE_FAIRNESS_MIN_TOTAL_SETTING, + WORKSPACE_REGISTRIES_SETTING, }, scripts::ScriptLang, stats_oss::schedule_stats, @@ -134,8 +137,11 @@ use crate::monitor::{ reload_job_default_timeout_setting, reload_job_isolation_setting, reload_jwt_secret_setting, reload_license_key, reload_npm_config_registry_setting, reload_nsjail_tmp_backing_setting, reload_nsjail_tmpfs_size_setting, reload_otel_tracing_proxy_setting, - reload_pip_index_url_setting, reload_retention_period_setting, reload_scim_token_setting, - reload_smtp_config, reload_store_audit_logs_s3_setting, reload_uv_exclude_newer_setting, + reload_pip_index_url_setting, reload_retention_period_setting, + reload_sandbox_image_cache_max_setting, reload_sandbox_image_default_registry_setting, + reload_sandbox_image_max_size_setting, reload_sandbox_image_pull_policy_setting, + reload_sandbox_registry_auth_setting, reload_scim_token_setting, reload_smtp_config, + reload_store_audit_logs_s3_setting, reload_uv_exclude_newer_setting, reload_uv_index_strategy_setting, reload_uv_python_install_mirror_setting, reload_worker_config, MonitorIteration, }; @@ -1827,6 +1833,19 @@ async fn process_notify_event( JOB_ISOLATION_SETTING => reload_job_isolation_setting(conn).await, NSJAIL_TMPFS_SIZE_MB_SETTING => reload_nsjail_tmpfs_size_setting(conn).await, NSJAIL_TMP_BACKING_SETTING => reload_nsjail_tmp_backing_setting(conn).await, + SANDBOX_IMAGE_MAX_SIZE_MB_SETTING => { + reload_sandbox_image_max_size_setting(conn).await + } + SANDBOX_IMAGE_CACHE_MAX_MB_SETTING => { + reload_sandbox_image_cache_max_setting(conn).await + } + SANDBOX_IMAGE_PULL_POLICY_SETTING => { + reload_sandbox_image_pull_policy_setting(conn).await + } + SANDBOX_IMAGE_DEFAULT_REGISTRY_SETTING => { + reload_sandbox_image_default_registry_setting(conn).await + } + SANDBOX_REGISTRY_AUTH_SETTING => reload_sandbox_registry_auth_setting(conn).await, #[cfg(feature = "parquet")] OBJECT_STORE_CONFIG_SETTING => { if !disable_s3_store { diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 13f52037e7..789706e7f8 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -66,7 +66,9 @@ use windmill_common::{ OTEL_SETTING, OTEL_TRACING_PROXY_SETTING, PIP_INDEX_URL_SETTING, POWERSHELL_REPO_PAT_SETTING, POWERSHELL_REPO_URL_SETTING, PREVIEW_TAGS_OVERRIDE_SETTING, REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, - RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, + RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SANDBOX_IMAGE_CACHE_MAX_MB_SETTING, + SANDBOX_IMAGE_DEFAULT_REGISTRY_SETTING, SANDBOX_IMAGE_MAX_SIZE_MB_SETTING, + SANDBOX_IMAGE_PULL_POLICY_SETTING, SANDBOX_REGISTRY_AUTH_SETTING, SCIM_TOKEN_SETTING, STORE_AUDIT_LOGS_S3_SETTING, TIMEOUT_WAIT_RESULT_SETTING, UV_EXCLUDE_NEWER_SETTING, UV_INDEX_STRATEGY_SETTING, UV_PYTHON_INSTALL_MIRROR_SETTING, WORKSPACE_FAIRNESS_DURATION_SECS_SETTING, WORKSPACE_FAIRNESS_ENABLED_SETTING, @@ -112,8 +114,10 @@ use windmill_worker::{ JOB_DEFAULT_TIMEOUT, JOB_ISOLATION, KEEP_JOB_DIR, MAVEN_REPOS, MAVEN_SETTINGS_XML, NO_DEFAULT_MAVEN, NPMRC, NPM_CONFIG_REGISTRY, NSJAIL_AVAILABLE, NSJAIL_TMPFS_SIZE_MB, NSJAIL_TMP_BACKING, NUGET_CONFIG, OTEL_TRACING_PROXY_SETTINGS, PIP_EXTRA_INDEX_URL, - PIP_INDEX_URL, POWERSHELL_REPO_PAT, POWERSHELL_REPO_URL, UNSHARE_PATH, UV_EXCLUDE_NEWER, - UV_INDEX_STRATEGY, UV_PYTHON_INSTALL_MIRROR, WORKSPACE_REGISTRIES, + PIP_INDEX_URL, POWERSHELL_REPO_PAT, POWERSHELL_REPO_URL, SANDBOX_IMAGE_CACHE_MAX_MB, + SANDBOX_IMAGE_DEFAULT_REGISTRY, SANDBOX_IMAGE_MAX_SIZE_MB, SANDBOX_IMAGE_PULL_POLICY, + SANDBOX_REGISTRY_AUTH, UNSHARE_PATH, UV_EXCLUDE_NEWER, UV_INDEX_STRATEGY, + UV_PYTHON_INSTALL_MIRROR, WORKSPACE_REGISTRIES, }; #[cfg(feature = "parquet")] @@ -407,6 +411,11 @@ pub async fn initial_load( reload_job_isolation_setting(&conn).await; reload_nsjail_tmpfs_size_setting(&conn).await; reload_nsjail_tmp_backing_setting(&conn).await; + reload_sandbox_image_max_size_setting(&conn).await; + reload_sandbox_image_cache_max_setting(&conn).await; + reload_sandbox_image_pull_policy_setting(&conn).await; + reload_sandbox_image_default_registry_setting(&conn).await; + reload_sandbox_registry_auth_setting(&conn).await; reload_extra_pip_index_url_setting(&conn).await; reload_pip_index_url_setting(&conn).await; reload_uv_index_strategy_setting(&conn).await; @@ -2045,6 +2054,66 @@ pub async fn reload_nsjail_tmp_backing_setting(conn: &Connection) { .await; } +pub async fn reload_sandbox_image_max_size_setting(conn: &Connection) { + reload_option_setting_with_tracing( + conn, + SANDBOX_IMAGE_MAX_SIZE_MB_SETTING, + "SANDBOX_IMAGE_MAX_SIZE_MB", + SANDBOX_IMAGE_MAX_SIZE_MB.clone(), + ) + .await; +} + +pub async fn reload_sandbox_image_cache_max_setting(conn: &Connection) { + reload_option_setting_with_tracing( + conn, + SANDBOX_IMAGE_CACHE_MAX_MB_SETTING, + "SANDBOX_IMAGE_CACHE_MAX_MB", + SANDBOX_IMAGE_CACHE_MAX_MB.clone(), + ) + .await; +} + +pub async fn reload_sandbox_image_pull_policy_setting(conn: &Connection) { + reload_option_setting_with_tracing( + conn, + SANDBOX_IMAGE_PULL_POLICY_SETTING, + "SANDBOX_IMAGE_PULL_POLICY", + SANDBOX_IMAGE_PULL_POLICY.clone(), + ) + .await; +} + +pub async fn reload_sandbox_image_default_registry_setting(conn: &Connection) { + reload_option_setting_with_tracing( + conn, + SANDBOX_IMAGE_DEFAULT_REGISTRY_SETTING, + "SANDBOX_IMAGE_DEFAULT_REGISTRY", + SANDBOX_IMAGE_DEFAULT_REGISTRY.clone(), + ) + .await; +} + +pub async fn reload_sandbox_registry_auth_setting(conn: &Connection) { + // Secret-aware: the value is a raw docker/podman auth.json with credentials, so + // it must never be logged. Load directly (the generic reload_option_setting path + // logs the value via load_option_setting_value) and only log a redacted message. + let q = + match load_value_from_global_settings_with_conn(conn, SANDBOX_REGISTRY_AUTH_SETTING, true) + .await + { + Ok(q) => q, + Err(e) => { + tracing::error!("Error reloading setting SANDBOX_REGISTRY_AUTH: {e:?}"); + return; + } + }; + let value = q.and_then(|q| serde_json::from_value::(q).ok()); + let configured = value.as_ref().is_some_and(|v| !v.trim().is_empty()); + *SANDBOX_REGISTRY_AUTH.write().await = value; + tracing::info!("Loaded setting SANDBOX_REGISTRY_AUTH (redacted), configured={configured}"); +} + pub async fn reload_job_isolation_setting(conn: &Connection) { let value = match load_value_from_global_settings_with_conn(conn, JOB_ISOLATION_SETTING, true).await { diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index 75f3fdf06b..8192f186d0 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -58,6 +58,11 @@ pub const NSJAIL_TMPFS_SIZE_MB_SETTING: &str = "nsjail_tmpfs_size_mb"; pub const NSJAIL_TMP_BACKING_SETTING: &str = "nsjail_tmp_backing"; pub const NSJAIL_TMP_BACKING_DISK: &str = "disk"; pub const NSJAIL_TMP_BACKING_TMPFS: &str = "tmpfs"; +pub const SANDBOX_IMAGE_MAX_SIZE_MB_SETTING: &str = "sandbox_image_max_size_mb"; +pub const SANDBOX_IMAGE_CACHE_MAX_MB_SETTING: &str = "sandbox_image_cache_max_mb"; +pub const SANDBOX_IMAGE_PULL_POLICY_SETTING: &str = "sandbox_image_pull_policy"; +pub const SANDBOX_IMAGE_DEFAULT_REGISTRY_SETTING: &str = "sandbox_image_default_registry"; +pub const SANDBOX_REGISTRY_AUTH_SETTING: &str = "sandbox_registry_auth"; pub const OBJECT_STORE_CONFIG_SETTING: &str = "object_store_cache_config"; pub const HUB_API_SECRET_SETTING: &str = "hub_api_secret"; diff --git a/backend/windmill-common/src/instance_config.rs b/backend/windmill-common/src/instance_config.rs index 2239868982..bb56d5d0cc 100644 --- a/backend/windmill-common/src/instance_config.rs +++ b/backend/windmill-common/src/instance_config.rs @@ -976,6 +976,7 @@ const SENSITIVE_SETTINGS: &[&str] = &[ "ruby_repos", "powershell_repo_pat", "workspace_registries", + "sandbox_registry_auth", ]; /// Object-valued settings that contain sensitive sub-fields. diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 92ebf08477..c7816d7a70 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -859,6 +859,37 @@ pub struct BashAnnotations { pub sandbox: bool, } +impl BashAnnotations { + /// If the script declares `# sandbox ` (an image ref after the sandbox + /// annotation), returns that image ref. This selects the daemonless, sandboxed + /// container runtime: extract the image's rootfs and run it inside the job's + /// nsjail sandbox. + /// + /// A bare `# sandbox` (no image argument) returns `None` and keeps the plain + /// nsjail-sandboxed-bash behavior (the `sandbox` boolean modifier). `# docker` + /// is unaffected and keeps the legacy v1 (dind/daemon) path. + pub fn sandbox_image(code: &str) -> Option { + for line in code.lines() { + let line = line.trim(); + if line.is_empty() { + continue; + } + // Mirror the annotation parser: stop at the first non-comment line. + if !line.starts_with('#') { + break; + } + let mut tokens = line[1..].split_whitespace(); + if tokens.next() == Some("sandbox") { + // `# sandbox ` -> container; bare `# sandbox` -> nsjail bash. + if let Some(image) = tokens.next() { + return Some(image.to_string()); + } + } + } + None + } +} + #[derive(Debug, Clone, Copy, PartialEq)] pub enum SqlResultCollectionStrategy { LastStatementAllRows, @@ -2224,6 +2255,34 @@ mod tests { use super::*; use std::collections::HashMap; + #[test] + fn test_bash_sandbox_image_annotation() { + // `# sandbox ` selects the container runtime and returns the image. + assert_eq!( + BashAnnotations::sandbox_image("# sandbox alpine:latest\necho hi"), + Some("alpine:latest".to_string()) + ); + // Extra whitespace and a leading non-spaced `#` still work. + assert_eq!( + BashAnnotations::sandbox_image("#sandbox python:3.12-slim\n"), + Some("python:3.12-slim".to_string()) + ); + // A bare `# sandbox` (no image) keeps the nsjail-bash modifier -> None. + assert_eq!(BashAnnotations::sandbox_image("# sandbox\necho hi"), None); + // `sandbox` must be its own token, not a prefix. + assert_eq!(BashAnnotations::sandbox_image("# sandboxed foo"), None); + // Stops at the first non-comment line (image declared too late is ignored). + assert_eq!( + BashAnnotations::sandbox_image("echo hi\n# sandbox alpine"), + None + ); + // `# docker` is a different annotation -> not a sandbox image. + assert_eq!( + BashAnnotations::sandbox_image("# docker alpine\necho hi"), + None + ); + } + #[test] fn test_mixed_tags() { let input = vec![ diff --git a/backend/windmill-worker/nsjail/run.docker.config.proto b/backend/windmill-worker/nsjail/run.docker.config.proto new file mode 100644 index 0000000000..a2da459fbe --- /dev/null +++ b/backend/windmill-worker/nsjail/run.docker.config.proto @@ -0,0 +1,103 @@ +name: "docker v2 run" + +mode: ONCE +hostname: "container" +log_level: ERROR +time_limit: {TIMEOUT} + +disable_rl: true + +cwd: {WORKDIR} + +clone_newnet: false +clone_newuser: {CLONE_NEWUSER} + +skip_setsid: true +keep_caps: false +# keep_env forwards nsjail's OWN process env (only windmill-trusted keys: reserved +# vars + proxy) to the child. The image's attacker-controlled Env is delivered via +# the envar directives below — NEVER nsjail's process env, so a hostile image cannot +# set LD_PRELOAD/LD_LIBRARY_PATH/LD_AUDIT on the nsjail binary itself. +keep_env: true +mount_proc: true + +# Image Env (+ PATH/HOME fallbacks), proto-escaped. Applied to the child only. +{ENVARS} + +# Map uid/gid 0 inside the jail to the (single) worker user outside. The image's +# rootfs is extracted as the worker user, so a root process inside the container +# owns the rootfs and runs like a normal "root in container" — without any subuid +# range. Multi-uid images are a later enhancement (newuidmap range). +uidmap { + inside_id: "0" + outside_id: "" + count: 1 +} +gidmap { + inside_id: "0" + outside_id: "" + count: 1 +} + +# The image's root filesystem, bound one top-level entry at a time. Binding the +# whole rootfs at "/" trips nsjail's read-only remount of its base root in a +# rootless userns ("mount(... MS_REMOUNT|MS_BIND|MS_RDONLY): Operation not +# permitted"); per-entry binds sit as rw submounts under nsjail's own tmpfs root +# and avoid it. Generated from the extracted rootfs. +{ROOTFS_MOUNTS} + +# Pseudo-filesystems the image expects. /tmp honors the same instance settings as +# every other nsjail job (nsjail_tmp_backing tmpfs/disk, nsjail_tmpfs_size_mb); +# /dev gets the standard nodes; /proc comes from mount_proc (the jail's own pid ns). +{TMP_MOUNT_BLOCK} + +mount { + src: "/dev/null" + dst: "/dev/null" + is_bind: true + rw: true +} + +mount { + src: "/dev/zero" + dst: "/dev/zero" + is_bind: true + rw: true +} + +mount { + src: "/dev/random" + dst: "/dev/random" + is_bind: true +} + +mount { + src: "/dev/urandom" + dst: "/dev/urandom" + is_bind: true +} + +# Host DNS config layered over the image's /etc so name resolution works on the +# job's network (mandatory:false: some minimal images have no /etc files to shadow). +mount { + src: "/etc/resolv.conf" + dst: "/etc/resolv.conf" + is_bind: true + mandatory: false +} + +mount { + src: "/etc/hosts" + dst: "/etc/hosts" + is_bind: true + mandatory: false +} + +# `# volume` mounts (and the same-worker /tmp/shared folder). Placed after the +# rootfs binds and the tmpfs /tmp so a volume target overrides any colliding image +# path and isn't shadowed by the tmpfs. Empty when there are no volumes. +{SHARED_MOUNT} + +iface_no_lo: true + +#{DEV} diff --git a/backend/windmill-worker/src/bash_executor.rs b/backend/windmill-worker/src/bash_executor.rs index 516c75fdba..4c470b8dfc 100644 --- a/backend/windmill-worker/src/bash_executor.rs +++ b/backend/windmill-worker/src/bash_executor.rs @@ -40,9 +40,9 @@ use crate::handle_child::run_future_with_polling_update_job_poller; use crate::{ common::{ - build_args_map, build_command_with_isolation, get_reserved_variables, read_file, - read_file_content, resolve_nsjail_timeout, resolve_nsjail_tmp_mount_block, start_child_process, - OccupancyMetrics, DEV_CONF_NSJAIL, + build_args_map, build_command_with_isolation, get_reserved_variables, raw_to_string, + read_file, read_file_content, resolve_nsjail_timeout, resolve_nsjail_tmp_mount_block, + start_child_process, OccupancyMetrics, DEV_CONF_NSJAIL, }, get_proxy_envs_for_lang, handle_child::handle_child, @@ -57,14 +57,6 @@ lazy_static::lazy_static! { pub static ref ANSI_ESCAPE_RE: Regex = Regex::new(r"\x1b\[[0-9;]*m").unwrap(); } -fn raw_to_string(x: &str) -> String { - match serde_json::from_str::(x) { - Ok(serde_json::Value::String(x)) => x, - Ok(x) => serde_json::to_string(&x).unwrap_or_else(|_| String::new()), - _ => String::new(), - } -} - #[tracing::instrument(level = "trace", skip_all)] pub async fn handle_bash_job( mem_peak: &mut i32, @@ -84,6 +76,28 @@ pub async fn handle_bash_job( ) -> Result, Error> { let annotation = windmill_common::worker::BashAnnotations::parse(&content); + // `# sandbox ` selects the daemonless, nsjail-sandboxed container runtime + // (extract the image's rootfs + run it inside the job's sandbox). A bare + // `# sandbox` keeps the plain nsjail-bash modifier; `# docker` keeps v1 (dind). + if let Some(image) = windmill_common::worker::BashAnnotations::sandbox_image(content) { + return crate::docker_v2::handle_docker_v2_job( + &image, + mem_peak, + canceled_by, + job, + conn, + client, + parent_runnable_path, + content, + job_dir, + shared_mount, + base_internal_url, + worker_name, + occupancy_metrics, + ) + .await; + } + // Check if sandbox annotation is used but nsjail is not available if annotation.sandbox && NSJAIL_AVAILABLE.is_none() { return Err(Error::ExecutionErr( diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index 9864093cd0..1bef9e4c6d 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -68,6 +68,16 @@ mount { #[cfg(not(debug_assertions))] pub const DEV_CONF_NSJAIL: &str = ""; +/// Turn a JSON value into the string a shell/CLI arg should receive: a JSON string +/// becomes its inner value, anything else is re-serialized compactly. +pub(crate) fn raw_to_string(x: &str) -> String { + match serde_json::from_str::(x) { + Ok(serde_json::Value::String(x)) => x, + Ok(x) => serde_json::to_string(&x).unwrap_or_else(|_| String::new()), + _ => String::new(), + } +} + pub async fn build_args_map<'a>( job: &'a MiniPulledJob, client: &AuthedClient, diff --git a/backend/windmill-worker/src/docker_v2.rs b/backend/windmill-worker/src/docker_v2.rs new file mode 100644 index 0000000000..6a99d252f3 --- /dev/null +++ b/backend/windmill-worker/src/docker_v2.rs @@ -0,0 +1,683 @@ +//! Sandboxed container runtime: run a container as a sandboxed subprogram of the job. +//! +//! Unlike the legacy `# docker` (dind/daemon) path, this has no daemon and no Docker +//! API. It splits *pull* from *run*: +//! +//! 1. **pull/extract** (podman, rootless): materialize the image's root filesystem +//! into `{job_dir}/rootfs` and read its OCI config (Env/Cmd/Entrypoint/WorkingDir). +//! 2. **run** (the job's own nsjail sandbox): execute the image command with the +//! extracted rootfs bound in as the new root, so the container inherits exactly +//! the job's confinement (filesystem mask, pid namespace, network, uid) and can't +//! escape past what the job itself can reach. +//! +//! Selected by `# sandbox ` (a bare `# sandbox` keeps plain nsjail-bash; +//! `# docker` keeps the v1 daemon path). The script body runs inside the image via +//! `/bin/sh`; an empty body runs the image's ENTRYPOINT/CMD. + +use std::process::Stdio; + +use serde::Deserialize; +use serde_json::{json, value::RawValue}; +use sqlx::types::Json; +use tokio::process::Command; + +use windmill_common::{client::AuthedClient, scripts::ScriptLang}; +use windmill_common::{ + error::Error, + worker::{to_raw_value, write_file, Connection}, +}; + +use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; + +use crate::{ + common::{ + build_args_map, get_reserved_variables, raw_to_string, resolve_nsjail_timeout, + resolve_nsjail_tmp_mount_block, start_child_process, OccupancyMetrics, DEV_CONF_NSJAIL, + }, + get_proxy_envs_for_lang, + handle_child::handle_child, + DISABLE_NUSER, NSJAIL_AVAILABLE, NSJAIL_PATH, SANDBOX_IMAGE_CACHE_MAX_MB, + SANDBOX_IMAGE_DEFAULT_REGISTRY, SANDBOX_IMAGE_MAX_SIZE_MB, SANDBOX_IMAGE_PULL_POLICY, + SANDBOX_REGISTRY_AUTH, +}; + +const NSJAIL_CONFIG_RUN_DOCKER_CONTENT: &str = include_str!("../nsjail/run.docker.config.proto"); + +const DEFAULT_PATH: &str = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"; + +lazy_static::lazy_static! { + pub static ref PODMAN_PATH: String = + std::env::var("PODMAN_PATH").unwrap_or_else(|_| "podman".to_string()); +} + +/// Guards against overlapping cache-eviction passes across concurrent jobs. +static EVICTION_RUNNING: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); + +/// podman pull policy from the `sandbox_image_pull_policy` instance setting. `newer` +/// (the default when unset/invalid) re-pulls only when the registry digest changed — +/// one cheap manifest check per job, no transfer if unchanged — so moving tags like +/// `:latest` don't go stale. `missing` is fastest (tags can go stale); `always` +/// re-checks every job. +async fn pull_policy() -> String { + let p = SANDBOX_IMAGE_PULL_POLICY.read().await.clone(); + match p.as_deref() { + Some(p @ ("missing" | "newer" | "always" | "never")) => p.to_string(), + _ => "newer".to_string(), + } +} + +/// `sandbox_image_max_size_mb` instance setting; 0 (or unset/non-positive) = no limit. +async fn max_image_size_mb() -> u64 { + SANDBOX_IMAGE_MAX_SIZE_MB.read().await.unwrap_or(0).max(0) as u64 +} + +/// `sandbox_image_cache_max_mb` instance setting; 0 (or unset/non-positive) = unbounded. +async fn image_cache_max_mb() -> u64 { + SANDBOX_IMAGE_CACHE_MAX_MB.read().await.unwrap_or(0).max(0) as u64 +} + +/// A ref is registry-qualified if the component before the first `/` looks like a +/// host (contains `.` or `:`, or is `localhost`). Bare repos (`alpine`, +/// `alpine:latest`, `myorg/img`) are unqualified and resolve against docker.io — +/// or the configured default registry. +fn registry_qualified(image: &str) -> bool { + match image.split_once('/') { + None => false, + Some((first, _)) => first.contains('.') || first.contains(':') || first == "localhost", + } +} + +/// Prepend the `sandbox_image_default_registry` instance setting to unqualified image +/// refs (fully-qualified refs are left untouched). +async fn resolve_image_ref(image: &str) -> String { + let registry = SANDBOX_IMAGE_DEFAULT_REGISTRY.read().await.clone(); + match registry { + Some(registry) if !registry.trim().is_empty() && !registry_qualified(image) => { + format!("{}/{}", registry.trim().trim_end_matches('/'), image) + } + _ => image.to_string(), + } +} + +/// If the `sandbox_registry_auth` instance setting holds a docker/podman `auth.json` +/// blob, write it to a per-job authfile (0600, removed with the job) and return its +/// path to pass to `podman --authfile`. Returns `None` when unset. +async fn write_auth_file(job_dir: &str) -> Result, Error> { + let auth = SANDBOX_REGISTRY_AUTH.read().await.clone(); + let Some(auth) = auth.filter(|a| !a.trim().is_empty()) else { + return Ok(None); + }; + let path = format!("{job_dir}/registry_auth.json"); + // Create 0600 from the start (registry credentials) — no world-readable window. + #[cfg(unix)] + { + use tokio::io::AsyncWriteExt; + let mut f = tokio::fs::OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .mode(0o600) + .open(&path) + .await?; + f.write_all(auth.as_bytes()).await?; + } + #[cfg(not(unix))] + tokio::fs::write(&path, auth).await?; + Ok(Some(path)) +} + +/// The subset of an image's OCI config we apply to the run. +#[derive(Deserialize, Default, Debug)] +struct OciConfig { + #[serde(default, rename = "Env")] + env: Option>, + #[serde(default, rename = "Cmd")] + cmd: Option>, + #[serde(default, rename = "Entrypoint")] + entrypoint: Option>, + #[serde(default, rename = "WorkingDir")] + working_dir: Option, +} + +/// Quote a string as a protobuf-text-format string literal for safe inclusion in +/// the nsjail config. Image-controlled values (mount srcs/dsts, symlink targets, +/// WorkingDir) flow into the config, so they MUST be escaped — an unescaped `"` or +/// newline would otherwise let a hostile image config inject arbitrary nsjail +/// directives and break out of the sandbox. Every byte is emitted as a printable +/// ASCII char or a valid protobuf escape (`\"`, `\\`, `\n`/`\r`/`\t`, or 3-digit +/// octal `\NNN` for control/non-ASCII bytes), so the result always parses. +fn proto_str(s: &str) -> String { + let mut out = String::with_capacity(s.len() + 2); + out.push('"'); + for &b in s.as_bytes() { + match b { + b'"' => out.push_str("\\\""), + b'\\' => out.push_str("\\\\"), + b'\n' => out.push_str("\\n"), + b'\r' => out.push_str("\\r"), + b'\t' => out.push_str("\\t"), + 0x20..=0x7e => out.push(b as char), + _ => out.push_str(&format!("\\{b:03o}")), + } + } + out.push('"'); + out +} + +/// Render container env vars as nsjail `envar:` directives (one per line). Each +/// `KEY=VALUE` is proto-escaped, so image-controlled keys/values can neither break +/// the config nor reach nsjail's own process environment. +fn render_envars(env: &[(String, String)]) -> String { + env.iter() + .map(|(k, v)| format!("envar: {}", proto_str(&format!("{k}={v}")))) + .collect::>() + .join("\n") +} + +async fn podman(args: &[&str]) -> Result { + Command::new(PODMAN_PATH.as_str()) + .args(args) + .output() + .await + .map_err(|e| Error::ExecutionErr(format!("failed to run podman {}: {e}", args.join(" ")))) +} + +/// Pull (if needed) and unpack `image` into `{job_dir}/rootfs`, returning its OCI +/// config. Uses podman rootless: `create` (auto-pulls) + `export | tar -x`, with the +/// config read from the resulting container (== image config, no command override). +async fn extract_image(image: &str, job_dir: &str) -> Result { + let rootfs = format!("{job_dir}/rootfs"); + tokio::fs::create_dir_all(&rootfs).await?; + + // `podman create` (no command) pulls the image per the configured policy and + // records the image's own Cmd/Entrypoint, which we then read back from the + // container config. `--` guards against an `image` ref that starts with `-` being + // parsed as a flag (e.g. `--authfile=...`) — the ref is attacker-controlled in + // the untrusted case. + let pull = format!("--pull={}", pull_policy().await); + let mut create_args = vec!["create", &pull]; + let authfile = write_auth_file(job_dir).await?; + if let Some(authfile) = authfile.as_deref() { + create_args.push("--authfile"); + create_args.push(authfile); + } + create_args.push("--"); + create_args.push(image); + let created = podman(&create_args).await?; + if !created.status.success() { + return Err(Error::ExecutionErr(format!( + "failed to pull/create image {image}: {}", + String::from_utf8_lossy(&created.stderr) + ))); + } + let container_id = String::from_utf8_lossy(&created.stdout).trim().to_string(); + + // Always clean up the container, even on a later failure. + let result = extract_created(image, &container_id, &rootfs).await; + let _ = podman(&["rm", "-f", &container_id]).await; + result +} + +async fn extract_created( + image: &str, + container_id: &str, + rootfs: &str, +) -> Result { + // Reject oversized images before paying the (large) extraction cost. + enforce_image_size_limit(image).await?; + + let inspected = podman(&["inspect", container_id, "--format", "{{json .Config}}"]).await?; + if !inspected.status.success() { + return Err(Error::ExecutionErr(format!( + "failed to inspect image {image}: {}", + String::from_utf8_lossy(&inspected.stderr) + ))); + } + let config: OciConfig = serde_json::from_slice(&inspected.stdout) + .map_err(|e| Error::ExecutionErr(format!("failed to parse image {image} config: {e}")))?; + + // Flatten the image's layers into a rootfs directory. Go through a tar on disk + // (in the job dir, cleaned up with the job) rather than a shell pipe. Extracted + // as the worker user, so the rootfs is owned by the worker user — which the + // single-uid jail maps to uid 0 inside. + let tar_path = format!("{rootfs}.tar"); + let exported = podman(&["export", container_id, "--output", &tar_path]).await?; + if !exported.status.success() { + let _ = tokio::fs::remove_file(&tar_path).await; + return Err(Error::ExecutionErr(format!( + "failed to export image {image}: {}", + String::from_utf8_lossy(&exported.stderr) + ))); + } + let untar = Command::new("tar") + .args(["-xf", &tar_path, "-C", rootfs]) + .output() + .await + .map_err(|e| Error::ExecutionErr(format!("failed to run tar: {e}")))?; + let _ = tokio::fs::remove_file(&tar_path).await; + if !untar.status.success() { + return Err(Error::ExecutionErr(format!( + "failed to unpack image {image}: {}", + String::from_utf8_lossy(&untar.stderr) + ))); + } + + Ok(config) +} + +/// Reject the image if its on-disk (uncompressed) size exceeds +/// `SANDBOX_IMAGE_MAX_SIZE_MB`. No-op when the limit is 0 (unset). +async fn enforce_image_size_limit(image: &str) -> Result<(), Error> { + let max = max_image_size_mb().await; + if max == 0 { + return Ok(()); + } + let out = podman(&["image", "inspect", image, "--format", "{{.Size}}"]).await?; + if !out.status.success() { + // Don't silently bypass the guard — surface it so an operator can see the + // size limit isn't being enforced for this image. + tracing::warn!( + "sandbox image size guard: `podman image inspect {image}` failed, not \ + enforcing SANDBOX_IMAGE_MAX_SIZE_MB: {}", + String::from_utf8_lossy(&out.stderr) + ); + return Ok(()); + } + let bytes: u64 = String::from_utf8_lossy(&out.stdout) + .trim() + .parse() + .unwrap_or(0); + let mb = bytes / 1_000_000; + if mb > max { + return Err(Error::ExecutionErr(format!( + "image {image} is {mb} MB, over the SANDBOX_IMAGE_MAX_SIZE_MB limit of {max} MB" + ))); + } + Ok(()) +} + +#[derive(Deserialize)] +struct PodmanImage { + #[serde(rename = "Id")] + id: String, + // `default`: podman tags Size/Created `omitempty`, so a degenerate image with a + // zero value drops the key — without this the whole array would fail to parse. + #[serde(default, rename = "Size")] + size: u64, + #[serde(default, rename = "Created")] + created: i64, +} + +/// Best-effort eviction: while the summed size of podman's images exceeds +/// `SANDBOX_IMAGE_CACHE_MAX_MB`, remove the oldest (by created time, an LRU proxy). +/// No-op when the limit is 0 (unset). Skipped if another pass is already running. +/// Images currently backing a container (e.g. a concurrent job mid-extract) fail +/// `rmi` and stop the pass, so in-use images are never removed. +async fn enforce_image_cache_limit() { + use std::sync::atomic::Ordering; + let max_mb = image_cache_max_mb().await; + if max_mb == 0 { + return; + } + if EVICTION_RUNNING + .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst) + .is_err() + { + return; + } + // Reset the guard on every exit path (incl. an early `break` or a panic), so a + // stuck flag can never permanently disable eviction until a worker restart. + struct ResetOnDrop; + impl Drop for ResetOnDrop { + fn drop(&mut self) { + EVICTION_RUNNING.store(false, std::sync::atomic::Ordering::SeqCst); + } + } + let _reset = ResetOnDrop; + let max_bytes = max_mb.saturating_mul(1_000_000); + loop { + let Ok(out) = podman(&["images", "--format", "json"]).await else { + break; + }; + if !out.status.success() { + break; + } + let mut imgs: Vec = match serde_json::from_slice(&out.stdout) { + Ok(v) => v, + Err(e) => { + // Don't silently disable eviction on a schema hiccup — surface it. + tracing::warn!( + "sandbox image cache eviction: cannot parse `podman images` json: {e}" + ); + break; + } + }; + let total: u64 = imgs.iter().map(|i| i.size).sum(); + if total <= max_bytes || imgs.is_empty() { + break; + } + imgs.sort_by_key(|i| i.created); + let victim = imgs[0].id.clone(); + match podman(&["rmi", &victim]).await { + Ok(rm) if rm.status.success() => { + tracing::info!("sandbox image cache eviction: removed {victim}"); + } + Ok(rm) => { + tracing::warn!( + "sandbox image cache eviction: cannot remove {victim} (in use?): {}", + String::from_utf8_lossy(&rm.stderr) + ); + break; + } + Err(_) => break, + } + } + // `_reset` drops here and clears EVICTION_RUNNING. +} + +/// Build the nsjail mount block that binds each top-level entry of the rootfs in +/// place. Binding the whole rootfs at `/` trips nsjail's read-only remount of its +/// base root in a rootless userns; per-entry binds avoid it. `proc`, `dev`, `tmp` +/// and `sys` are skipped — the profile provides them. +async fn generate_rootfs_mounts(rootfs: &str) -> Result { + let mut block = String::new(); + let mut entries = tokio::fs::read_dir(rootfs).await?; + while let Some(entry) = entries.next_entry().await? { + let name = entry.file_name(); + let name = name.to_string_lossy(); + if matches!(name.as_ref(), "proc" | "dev" | "tmp" | "sys") { + continue; + } + let src = proto_str(&format!("{rootfs}/{name}")); + let dst = proto_str(&format!("/{name}")); + let file_type = entry.file_type().await?; + if file_type.is_symlink() { + // Recreate top-level symlinks (e.g. usr-merged /bin -> usr/bin) as + // symlinks in the jail. The target is image-controlled but only ever + // *resolved inside the jail* (against the bound rootfs dirs / jail + // pseudo-fs) — there is no host `/` in the jail for it to point at — and + // it is escaped via proto_str, so it can neither escape nor inject config. + let target = tokio::fs::read_link(entry.path()) + .await + .map(|p| p.to_string_lossy().into_owned()) + .unwrap_or_default(); + block.push_str(&format!( + "mount {{\n src: {}\n dst: {dst}\n is_symlink: true\n mandatory: false\n}}\n", + proto_str(&target), + )); + } else { + block.push_str(&format!( + "mount {{\n src: {src}\n dst: {dst}\n is_bind: true\n rw: true\n mandatory: false\n}}\n", + )); + } + } + Ok(block) +} + +#[tracing::instrument(level = "trace", skip_all)] +pub async fn handle_docker_v2_job( + image: &str, + mem_peak: &mut i32, + canceled_by: &mut Option, + job: &MiniPulledJob, + conn: &Connection, + client: &AuthedClient, + parent_runnable_path: Option, + content: &str, + job_dir: &str, + shared_mount: &str, + base_internal_url: &str, + worker_name: &str, + occupancy_metrics: &mut OccupancyMetrics, +) -> Result, Error> { + // The sandboxed container runtime *is* nsjail, so it requires nsjail. (`# docker` + // keeps the v1 dind path for non-sandboxed workers.) + if NSJAIL_AVAILABLE.is_none() { + return Err(Error::ExecutionErr(format!( + "`# sandbox {image}` runs the image inside nsjail, which is not available on \ + this worker. Install nsjail, or use a bare `# docker` (dind) instead." + ))); + } + + // Apply the default-registry instance setting to unqualified refs. + let resolved_image = resolve_image_ref(image).await; + let image = resolved_image.as_str(); + + append_logs( + &job.id, + &job.workspace_id, + format!("\n\n--- SANDBOXED CONTAINER (nsjail) ---\nextracting image {image}...\n"), + conn, + ) + .await; + + let config = extract_image(image, job_dir).await?; + let rootfs = format!("{job_dir}/rootfs"); + + // Best-effort: keep podman's image store under its size cap (overlaps the run). + tokio::spawn(enforce_image_cache_limit()); + + // Resolve the script args from the bash signature, like the bash executor. + let args = build_args_map(job, client, conn).await?.map(Json); + let job_args = if args.is_some() { + args.as_ref() + } else { + job.args.as_ref() + }; + let args_owned = windmill_parser_bash::parse_bash_sig(content)? + .args + .iter() + .map(|arg| { + job_args + .and_then(|x| x.get(&arg.name).map(|x| raw_to_string(x.get()))) + .unwrap_or_else(String::new) + }) + .collect::>(); + + // The body is everything that isn't a leading `#` annotation/comment line. With + // a body we run it via the image's `/bin/sh`; without one we run the image's + // ENTRYPOINT + CMD. + let has_body = content + .lines() + .any(|l| !l.trim().is_empty() && !l.trim_start().starts_with('#')); + + let cmd_args: Vec = if has_body { + // Pass the body straight to `sh -c` rather than writing a script file into + // the image-controlled rootfs: a malicious image could plant that path as a + // symlink to a host file and capture the worker's write before nsjail starts + // (sandbox-boundary bypass). `sh -c sh ` binds args as $1.. . + let mut v = vec![ + "/bin/sh".to_string(), + "-c".to_string(), + format!("set -e\n{content}"), + "sh".to_string(), + ]; + v.extend(args_owned.iter().cloned()); + v + } else { + let mut v = config.entrypoint.clone().unwrap_or_default(); + v.extend(config.cmd.clone().unwrap_or_default()); + if v.is_empty() { + return Err(Error::ExecutionErr(format!( + "image {image} has no ENTRYPOINT/CMD and the script body is empty — \ + nothing to run" + ))); + } + v.extend(args_owned.iter().cloned()); + v + }; + + let working_dir = config + .working_dir + .as_deref() + .filter(|w| !w.is_empty()) + .unwrap_or("/"); + + // The image's OCI Env is attacker-controlled (BOTH keys and values), so it must + // NOT enter the nsjail launcher's own process env: a hostile image could set + // LD_PRELOAD / LD_LIBRARY_PATH / LD_AUDIT and have the dynamic loader run code in + // the nsjail binary as the worker — outside the jail — before it sandboxes. + // Deliver it to the *child only* via proto-escaped `envar:` directives. + let mut container_env: Vec<(String, String)> = Vec::new(); + for kv in config.env.unwrap_or_default() { + if let Some((k, v)) = kv.split_once('=') { + container_env.push((k.to_string(), v.to_string())); + } + } + if !container_env.iter().any(|(k, _)| k == "PATH") { + container_env.push(("PATH".to_string(), DEFAULT_PATH.to_string())); + } + if !container_env.iter().any(|(k, _)| k == "HOME") { + container_env.push(("HOME".to_string(), "/root".to_string())); + } + let envars = render_envars(&container_env); + + // Render the nsjail profile: dynamic per-entry rootfs binds + image WorkingDir. + let nsjail_timeout = resolve_nsjail_timeout(conn, &job.workspace_id, job.id, job.timeout).await; + let rootfs_mounts = generate_rootfs_mounts(&rootfs).await?; + write_file( + job_dir, + "run.docker.config.proto", + &NSJAIL_CONFIG_RUN_DOCKER_CONTENT + .replace("{TIMEOUT}", &nsjail_timeout) + .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) + // proto_str-quoted: WorkingDir is image-controlled, must not break out + // of the `cwd:` string and inject nsjail directives. + .replace("{WORKDIR}", &proto_str(working_dir)) + .replace("{ROOTFS_MOUNTS}", &rootfs_mounts) + .replace( + "{TMP_MOUNT_BLOCK}", + &resolve_nsjail_tmp_mount_block(job_dir).await, + ) + // `# volume` mounts + same-worker shared folder (empty if none). + .replace("{SHARED_MOUNT}", shared_mount) + // Image env as `envar:` directives (child-only), so it never touches + // nsjail's process env. + .replace("{ENVARS}", &envars) + .replace("#{DEV}", DEV_CONF_NSJAIL), + )?; + + // nsjail's OWN process env: only windmill-trusted keys (reserved vars so + // `wmill`/API calls work, + proxy). `keep_env: true` forwards these to the + // child. The image env is NOT here — see container_env above. + let mut reserved_variables = + get_reserved_variables(job, &client.token, conn, parent_runnable_path).await?; + reserved_variables.insert("RUST_LOG".to_string(), "info".to_string()); + reserved_variables.insert( + "BASE_INTERNAL_URL".to_string(), + base_internal_url.to_string(), + ); + + let proxy_envs = get_proxy_envs_for_lang( + &ScriptLang::Bash, + job.kind, + &job.id, + &job.workspace_id, + conn, + ) + .await?; + + let mut nsjail_run_args = vec!["--config", "run.docker.config.proto", "--"]; + nsjail_run_args.extend(cmd_args.iter().map(|s| s.as_str())); + + let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); + nsjail_cmd + .current_dir(job_dir) + .env_clear() + .envs(reserved_variables) + .envs(proxy_envs) + .args(nsjail_run_args) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + let child = start_child_process(nsjail_cmd, NSJAIL_PATH.as_str(), false).await?; + + handle_child( + &job.id, + conn, + mem_peak, + canceled_by, + child, + true, + worker_name, + &job.workspace_id, + "sandboxed container run", + job.timeout, + true, + &mut Some(occupancy_metrics), + None, + None, + ) + .await?; + + Ok(to_raw_value(&json!(format!( + "sandboxed container ({image}) completed successfully" + )))) +} + +#[cfg(test)] +mod tests { + use super::{proto_str, registry_qualified, render_envars}; + + #[test] + fn render_envars_emits_proto_directives() { + // Image-controlled env (incl. loader vars) is rendered as `envar:` directives + // — i.e. delivered to the child via the config, NOT nsjail's process env, so + // it can never set LD_PRELOAD/etc. on the nsjail binary itself. + let env = vec![ + ("PATH".to_string(), "/usr/bin".to_string()), + ("LD_PRELOAD".to_string(), "rootfs/evil.so".to_string()), + ]; + let out = render_envars(&env); + assert_eq!( + out, + "envar: \"PATH=/usr/bin\"\nenvar: \"LD_PRELOAD=rootfs/evil.so\"" + ); + // A value trying to inject extra directives is escaped, not interpreted. + let evil = vec![("X".to_string(), "v\"\nclone_newuser: false".to_string())]; + let line = render_envars(&evil); + assert!(line.starts_with("envar: \"")); + assert!(!line.contains("\nclone_newuser")); + assert!(line.contains("\\n")); + } + + #[test] + fn proto_str_escapes_injection() { + // Normal paths are just wrapped in quotes. + assert_eq!(proto_str("/app"), "\"/app\""); + // A `"` is escaped so it cannot close the surrounding string and inject + // subsequent nsjail directives — this is what the WorkingDir / mount-src + // sandboxing fixes depend on. + let malicious = "/x\"\nmount { src: \"/\" dst: \"/host\" is_bind: true }\n#"; + let escaped = proto_str(malicious); + assert!(escaped.starts_with('"') && escaped.ends_with('"')); + // No raw quote or newline survives inside the rendered literal. + let inner = &escaped[1..escaped.len() - 1]; + assert!(!inner.contains('\n')); + assert!(!inner.contains("\"") || inner.contains("\\\"")); + assert!(escaped.contains("\\\"")); // the inner quote is backslash-escaped + assert!(escaped.contains("\\n")); // the newline is escaped + // Control and non-ASCII bytes render as valid 3-digit octal escapes (never + // a raw byte or an invalid `\u{..}` that nsjail's parser would reject). + assert_eq!(proto_str("a\u{1b}b"), "\"a\\033b\""); // ESC (0x1b) + assert_eq!(proto_str("é"), "\"\\303\\251\""); // UTF-8 bytes 0xc3 0xa9 + } + + #[test] + fn registry_qualified_classifies_refs() { + // Unqualified: bare repos (with/without tag) and docker.io org/repo. + for img in ["alpine", "alpine:latest", "myorg/img", "myorg/img:1.2"] { + assert!(!registry_qualified(img), "{img} should be unqualified"); + } + // Qualified: the first path component is a host (has `.`/`:`) or localhost. + for img in [ + "ghcr.io/org/img", + "registry.example.com/img:tag", + "localhost:5000/img", + "localhost/img", + "host:5000/a/b", + ] { + assert!(registry_qualified(img), "{img} should be qualified"); + } + } +} diff --git a/backend/windmill-worker/src/lib.rs b/backend/windmill-worker/src/lib.rs index 08f9381205..7727982cf8 100644 --- a/backend/windmill-worker/src/lib.rs +++ b/backend/windmill-worker/src/lib.rs @@ -31,6 +31,7 @@ mod csharp_executor; mod dedicated_worker_ee; mod dedicated_worker_oss; mod deno_executor; +mod docker_v2; #[cfg(feature = "duckdb")] mod duckdb_executor; mod global_cache; diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 7fb0dee9f5..b7ae7c2eed 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -694,6 +694,27 @@ lazy_static::lazy_static! { /// RAM-backed tmpfs sized by `nsjail_tmpfs_size_mb`. pub static ref NSJAIL_TMP_BACKING: Arc>> = Arc::new(RwLock::new(None)); + /// Reject a `# sandbox ` whose on-disk size exceeds this many MB, before + /// extraction. `None`/non-positive = no limit. (`sandbox_image_max_size_mb`.) + pub static ref SANDBOX_IMAGE_MAX_SIZE_MB: Arc>> = Arc::new(RwLock::new(None)); + + /// Best-effort cap (MB) on podman's sandbox-image store; oldest images evicted + /// after a run when exceeded. `None`/non-positive = unbounded. (`sandbox_image_cache_max_mb`.) + pub static ref SANDBOX_IMAGE_CACHE_MAX_MB: Arc>> = Arc::new(RwLock::new(None)); + + /// podman pull policy for sandbox images (`missing`/`newer`/`always`/`never`). + /// `None`/unrecognized falls back to `newer`. (`sandbox_image_pull_policy`.) + pub static ref SANDBOX_IMAGE_PULL_POLICY: Arc>> = Arc::new(RwLock::new(None)); + + /// If set, unqualified sandbox image refs (e.g. `alpine`) are pulled from this + /// registry instead of docker.io. Fully-qualified refs are unaffected. + /// (`sandbox_image_default_registry`.) + pub static ref SANDBOX_IMAGE_DEFAULT_REGISTRY: Arc>> = Arc::new(RwLock::new(None)); + + /// Optional docker/podman `auth.json` blob for private registries, written to a + /// per-job authfile and passed to `podman --authfile`. (`sandbox_registry_auth`.) + pub static ref SANDBOX_REGISTRY_AUTH: Arc>> = Arc::new(RwLock::new(None)); + /// Optional mirror URL for `uv python install`. Wires to the `UV_PYTHON_INSTALL_MIRROR` /// env var when forwarded to uv. Can be set via the `UV_PYTHON_INSTALL_MIRROR` env var /// or the `uv_python_install_mirror` instance setting. diff --git a/docker-compose.yml b/docker-compose.yml index 8b636c7702..75252cb802 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -49,25 +49,6 @@ services: logging: *default-logging - # Docker-in-Docker sidecar: provides an isolated Docker daemon so user scripts - # can run containers without accessing the host Docker socket. - dind: - image: docker:dind - privileged: true - restart: unless-stopped - environment: - DOCKER_TLS_CERTDIR: "" - volumes: - - dind-data:/var/lib/docker - expose: - - 2375 - healthcheck: - test: ["CMD", "docker", "info"] - interval: 10s - timeout: 5s - retries: 5 - logging: *default-logging - windmill_worker: image: ${WM_IMAGE} pull_policy: always @@ -89,22 +70,19 @@ services: # If running with non-root/non-windmill UID (e.g., user: "1001:1001"), # add: - HOME=/tmp - FAVOR_UNSHARE_PID=true - # Connect to the dind sidecar instead of the host Docker socket - - DOCKER_HOST=tcp://dind:2375 depends_on: db: condition: service_healthy - dind: - condition: service_healthy # to mount the worker folder to debug, KEEP_JOB_DIR=true and mount /tmp/windmill volumes: - worker_dependency_cache:/tmp/windmill/cache - worker_logs:/tmp/windmill/logs - ## WARNING: mounting the host Docker socket grants user scripts full access to - ## the host Docker daemon, enabling host filesystem access and privilege escalation. - ## Only use this if you fully trust all users who can run scripts. - ## To use it, remove the DOCKER_HOST env var and dind depends_on above, - ## and uncomment the line below: + ## Sandboxed containers (`# sandbox `) run daemonless via podman + nsjail + ## inside the worker itself — no Docker socket or dind sidecar required. + ## For the legacy full-compat docker (a bare `# docker`, trusted users only), + ## mount the host Docker socket by uncommenting the line below. WARNING: this + ## grants user scripts full access to the host Docker daemon (host filesystem + ## access and privilege escalation) — only use it if you fully trust all users. # - /var/run/docker.sock:/var/run/docker.sock logging: *default-logging @@ -237,4 +215,3 @@ volumes: windmill_index: null lsp_cache: null caddy_data: null - dind-data: null diff --git a/docs/docker-v2-runtime.md b/docs/docker-v2-runtime.md new file mode 100644 index 0000000000..c981e89b69 --- /dev/null +++ b/docs/docker-v2-runtime.md @@ -0,0 +1,106 @@ +# Sandboxed container runtime (daemonless docker) + +Windmill bash scripts can run a container image. There are **two** runtimes: + +| | legacy `# docker` | sandboxed `# sandbox ` | +|---|---|---| +| selected by | bare `# docker` | `# sandbox ` | +| runtime | dind / Docker daemon (bollard, `dind` feature) | daemonless: extract rootfs + nsjail-run | +| boundary | separate (daemon outside the jail) | the job's own nsjail sandbox | +| nsjail | not provided (trusted-tenant) | **required** — this *is* the sandbox | +| safety | trusted-tenant | sandboxed (untrusted-capable) | +| compat | full `docker run`/`-d`/API | run-a-command subset | + +The three bash annotations are distinct and don't overload each other: + +- `# docker` → legacy daemon docker (unchanged). +- `# sandbox` → run the bash script under nsjail. +- `# sandbox ` → run that image's command under nsjail (this runtime). + +## Using it + +Put the image ref on a `# sandbox` annotation line; the rest of the script runs +**inside** that image: + +```bash +# sandbox python:3.12-slim +name="$1" # windmill args bind positionally, like any bash script +python3 -c "import sys; print('hello', sys.argv[1])" "$name" +``` + +- The body runs via the image's `/bin/sh -c` (so the image needs a shell). +- An **empty** body runs the image's `ENTRYPOINT` + `CMD`. +- Windmill args (declared `x="$1"`, …) are appended to the command. +- The image's `Env`, `WorkingDir` are applied; the windmill reserved variables + (`WM_TOKEN`, `BASE_INTERNAL_URL`, …) are injected so `wmill`/API calls work. + +## How it works + +1. **Pull/extract** (podman, rootless): `podman create --pull= ` + + `podman export | tar -x` materializes the image's flattened root filesystem + into `{job_dir}/rootfs`, and `podman inspect` reads its OCI config. podman's + image store dedups pulls across jobs. +2. **Run** (the job's nsjail sandbox): nsjail binds each top-level entry of the + rootfs in place (binding the whole rootfs at `/` trips nsjail's read-only + remount of its base root in a rootless userns), mounts the standard + pseudo-filesystems (`/proc` from the jail's pid namespace, a tmpfs `/tmp`, + `/dev` nodes), maps uid/gid 0 inside → the worker user outside, and runs the + command. The container *is* the jail. + +``` +# sandbox ─▶ podman create+export ─▶ {job_dir}/rootfs ─▶ nsjail (chroot rootfs) + podman inspect (OCI config) ──────────────────▶ Env / Cmd / WorkingDir +``` + +Because the run is just the job's own nsjail with the image's filesystem as root, +the container inherits exactly the job's confinement: + +- **Filesystem**: only the rootfs + the job's mounts are visible — no host `/`, + no other job dirs, no dep cache. There is nothing to bind-mount escape to. +- **/proc**: the jail's own pid namespace — the worker and other jobs aren't + visible. +- **uid**: a single-uid jail — an escape lands as the unprivileged worker user. +- **network**: the job's network (same as any bash job). + +## Image storage, freshness & limits + +- **Where pulls live:** podman's rootless graph root (default + `$HOME/.local/share/containers/storage`) — persistent, dedups pulls across jobs. + The per-job extracted rootfs lives in `{job_dir}/rootfs` and is removed with the + job; the transient `rootfs.tar` is removed right after extraction. +- **Freshness (`SANDBOX_IMAGE_PULL_POLICY`, default `newer`):** `newer` re-pulls + only when the registry digest changed (one cheap manifest check per job, no data + transfer if unchanged) — so moving tags like `:latest` don't go stale. `missing` + is fastest but tags can go stale; `always` re-checks every job. Pinning a digest + (`img@sha256:…`) is immutable and never stale. +- **Per-image size cap (`SANDBOX_IMAGE_MAX_SIZE_MB`, default 0 = off):** images + whose on-disk size exceeds the cap are rejected before extraction. +- **Cache size cap (`SANDBOX_IMAGE_CACHE_MAX_MB`, default 0 = off):** best-effort + LRU eviction — after a run, the oldest images are removed until podman's image + store is back under the cap. In-use images are never removed. + +## Requirements + +- `podman` (rootless) and `tar` on the worker for image pull/extract. +- `nsjail` on the worker — **required**. If nsjail is absent, a `# sandbox ` + job errors clearly (use a bare `# docker` + a daemon instead). + +## Limitations (by design — daemonless, run-to-completion) + +- No `docker run -d` + later `exec`/`attach`/`logs -f`, no `docker build`, + `compose`, swarm, healthchecks. +- No arbitrary `-v` host bind mounts, `--privileged`, `--cap-add`, `--device`, + host namespace sharing. +- Images that drop to a non-root uid or chown to arbitrary uids inside need a + subuid **range** in the jail (single-uid only today — follow-up: `newuidmap` + range mapping). +- The script result is a completion message; capture output via stdout/logs. + +## Follow-ups + +- Content-addressed rootfs cache keyed by image digest (today each job re-exports; + podman's image store still dedups the network pull). +- Pre-pull size guard via `skopeo` manifest inspection (reject before download). +- Subuid-range nsjail variant for multi-uid images. +- Per-container isolated networking (slirp/pasta). +- Support under the non-nsjail `unshare` isolation mode. diff --git a/frontend/src/lib/components/ScriptBuilder.svelte b/frontend/src/lib/components/ScriptBuilder.svelte index 8f4e75ff3d..325f2753e5 100644 --- a/frontend/src/lib/components/ScriptBuilder.svelte +++ b/frontend/src/lib/components/ScriptBuilder.svelte @@ -1007,21 +1007,6 @@ function onScriptLanguageTrigger(lang: 'docker' | 'bunnative' | ScriptLang) { if (lang == 'docker') { - if (isCloudHosted()) { - sendUserToast( - 'You cannot use Docker scripts on the multi-tenant platform. Use a dedicated instance or self-host windmill instead.', - true, - [ - { - label: 'Learn more', - callback: () => { - window.open('https://www.windmill.dev/docs/advanced/docker', '_blank') - } - } - ] - ) - return - } template = 'docker' } else if (lang == 'bunnative') { template = 'bunnative' diff --git a/frontend/src/lib/components/flows/content/FlowInputs.svelte b/frontend/src/lib/components/flows/content/FlowInputs.svelte index 353dbb3a2f..335a37b0c7 100644 --- a/frontend/src/lib/components/flows/content/FlowInputs.svelte +++ b/frontend/src/lib/components/flows/content/FlowInputs.svelte @@ -7,8 +7,6 @@ import FlowScriptPicker from '../pickers/FlowScriptPicker.svelte' import PickHubScript from '../pickers/PickHubScript.svelte' import WorkspaceScriptPicker from '../pickers/WorkspaceScriptPicker.svelte' - import { isCloudHosted } from '$lib/cloud' - import { sendUserToast } from '$lib/toast' import ToggleButtonGroup from '$lib/components/common/toggleButton-v2/ToggleButtonGroup.svelte' import ToggleButton from '$lib/components/common/toggleButton-v2/ToggleButton.svelte' import { Check, Code, Zap } from 'lucide-svelte' @@ -259,23 +257,6 @@ {label} lang={lang == 'docker' ? 'bash' : lang} on:click={() => { - if (lang == 'docker') { - if (isCloudHosted()) { - sendUserToast( - 'You cannot use Docker scripts on the multi-tenant platform. Use a dedicated instance or self-host windmill instead.', - true, - [ - { - label: 'Learn more', - callback: () => { - window.open('https://www.windmill.dev/docs/advanced/docker', '_blank') - } - } - ] - ) - return - } - } dispatch('new', { language: lang == 'docker' ? 'bash' : lang, kind, diff --git a/frontend/src/lib/components/flows/content/FlowInputsQuick.svelte b/frontend/src/lib/components/flows/content/FlowInputsQuick.svelte index db199c745a..3f070dd74e 100644 --- a/frontend/src/lib/components/flows/content/FlowInputsQuick.svelte +++ b/frontend/src/lib/components/flows/content/FlowInputsQuick.svelte @@ -4,7 +4,6 @@