From dcee9fe7b163993836691988a552a5bc6042b9a2 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 16 Jan 2026 20:55:01 +0000 Subject: [PATCH 01/19] feat: cli branch override (#7592) * fix(cli): prevent duplicate 'Using non-dotted paths' log message Add a flag to track whether the message has already been logged, so it only prints once even if setNonDottedPaths is called multiple times. Co-Authored-By: Claude Opus 4.5 * feat(cli): add --branch option to sync pull/push commands Add a --branch argument that allows overriding the current git branch for sync operations. This enables: - Using branch-specific settings even when not in a git repository - Overriding the detected git branch when needed The branch override is applied to: - getEffectiveSettings() for branch-specific config overrides - getSpecificItemsForCurrentBranch() for branch-specific items Co-Authored-By: Claude Opus 4.5 * fix(cli): correct log message for workspace fork branches Use rawGitBranch instead of currentBranch in the log message when showing the origin of a workspace fork branch. Co-Authored-By: Claude Opus 4.5 * feat(cli): extend --branch support to specificItems functionality Update getBranchSpecificPath and isCurrentBranchFile to accept optional branchOverride parameter. This ensures that branch-specific file handling (for variables, resources, triggers) works correctly with --branch flag. Updated functions: - getBranchSpecificPath(): now accepts branchOverride - isCurrentBranchFile(): now accepts branchOverride - elementsToMap(): now accepts branchOverride - compareDynFSElement(): now accepts branchOverride - prettyChanges(): now accepts branchOverride All call sites updated to pass opts.branch through the call chain. Co-Authored-By: Claude Opus 4.5 * fix(cli): resolve TypeScript type errors - Fix Timeout type in dev.ts using ReturnType - Add proper type casts for unknown error types - Cast FlowModule to any to resolve generated type mismatch - Cast Uint8Array to BlobPart for Blob constructor compatibility Co-Authored-By: Claude Opus 4.5 * test(cli): add unit tests for branch detection and --branch override - Add specific_items.test.ts with 35 tests covering: - toBranchSpecificPath and fromBranchSpecificPath conversions - isSpecificItem pattern matching - isBranchSpecificFile detection - Round-trip path conversions - branchOverride parameter functionality - Add conf_branch_override.test.ts with 6 tests covering: - getEffectiveSettings with branchOverride parameter - Branch-specific overrides application - promotionOverrides precedence - Fallback to top-level settings - Fix containerized_backend.ts to use dynamic paths instead of hardcoded user home directories Co-Authored-By: Claude Opus 4.5 * chore(cli): add conf.ts barrel file for test imports Re-exports from src/core/conf.ts to support existing test imports. Co-Authored-By: Claude Opus 4.5 * all * fix(cli): pass --branch override to workspace resolution Previously, the --branch flag was only used for config resolution but not for workspace resolution. This caused confusing log messages that showed the git branch (e.g., master) before showing the override branch. Changes: - Add branchOverride parameter to tryResolveBranchWorkspace() - Add branchOverride parameter to resolveWorkspace() - Pass opts.branch from sync pull/push to resolveWorkspace() - Log "Using branch override" early in workspace resolution - Remove duplicate log from getEffectiveSettings() Now when using --branch foobar, the logs show: Using branch override: foobar Applied settings for Git branch: foobar ... Instead of the confusing previous output that mentioned both master and foobar. Co-Authored-By: Claude Opus 4.5 * refactor(cli): remove redundant branch detection in elementsToMap isCurrentBranchFile() already validates that a branch exists (via branchOverride or git detection) before returning true. No need to pre-compute currentBranch before calling it. Co-Authored-By: Claude Opus 4.5 --------- Co-authored-by: Claude Opus 4.5 --- cli/deno.lock | 6 + cli/src/commands/app/dev.ts | 4 +- cli/src/commands/init/init.ts | 2 +- cli/src/commands/schedule/schedule.ts | 4 +- cli/src/commands/script/script.ts | 4 +- cli/src/commands/sync/sync.ts | 63 +++-- cli/src/commands/trigger/trigger.ts | 4 +- cli/src/commands/workspace/fork.ts | 2 +- cli/src/commands/workspace/workspace.ts | 2 +- cli/src/core/conf.ts | 111 ++++---- cli/src/core/context.ts | 67 +++-- cli/src/core/specific_items.ts | 38 ++- cli/src/utils/resource_folders.ts | 4 +- cli/test/conf_branch_override.test.ts | 144 ++++++++++ cli/test/containerized_backend.ts | 13 +- cli/test/specific_items.test.ts | 338 ++++++++++++++++++++++++ 16 files changed, 680 insertions(+), 126 deletions(-) create mode 100644 cli/test/conf_branch_override.test.ts create mode 100644 cli/test/specific_items.test.ts diff --git a/cli/deno.lock b/cli/deno.lock index 53d3deaa04..3bbbf4a00a 100644 --- a/cli/deno.lock +++ b/cli/deno.lock @@ -26,6 +26,7 @@ "jsr:@std/internal@^1.0.12": "1.0.12", "jsr:@std/io@*": "0.225.2", "jsr:@std/io@0.223": "0.223.0", + "jsr:@std/io@~0.224.2": "0.224.9", "jsr:@std/io@~0.225.2": "0.225.2", "jsr:@std/log@*": "0.224.14", "jsr:@std/path@*": "1.1.3", @@ -169,6 +170,9 @@ "jsr:@std/bytes@0.223" ] }, + "@std/io@0.224.9": { + "integrity": "4414664b6926f665102e73c969cfda06d2c4c59bd5d0c603fd4f1b1c840d6ee3" + }, "@std/io@0.225.2": { "integrity": "3c740cd4ee4c082e6cfc86458f47e2ab7cb353dc6234d5e9b1f91a2de5f4d6c7", "dependencies": [ @@ -231,6 +235,7 @@ "dependencies": [ "jsr:@std/encoding@1.0.0-rc.2", "jsr:@std/fmt@~0.225.4", + "jsr:@std/io@~0.224.2", "jsr:@windmill-labs/cliffy-internal" ] }, @@ -261,6 +266,7 @@ "dependencies": [ "jsr:@std/assert@1.0.0-rc.2", "jsr:@std/fmt@~0.225.4", + "jsr:@std/io@~0.224.2", "jsr:@std/path@1.0.0-rc.2", "jsr:@std/text", "jsr:@windmill-labs/cliffy-ansi", diff --git a/cli/src/commands/app/dev.ts b/cli/src/commands/app/dev.ts index aa279cc779..c2412fe282 100644 --- a/cli/src/commands/app/dev.ts +++ b/cli/src/commands/app/dev.ts @@ -545,7 +545,7 @@ async function dev(opts: DevOptions) { runnablesWatcher = Deno.watchFs(runnablesPath); // Per-file debounce timeouts for schema inference (longer debounce for typing) - const schemaInferenceTimeouts: Record = {}; + const schemaInferenceTimeouts: Record> = {}; const SCHEMA_DEBOUNCE_MS = 500; // Wait 500ms after last change before inferring schema // Handle runnables file changes in the background @@ -1221,7 +1221,7 @@ async function dev(opts: DevOptions) { sqlWatcher = Deno.watchFs(sqlToApplyPath); // Debounce timeout for SQL file changes - const sqlDebounceTimeouts: Record = {}; + const sqlDebounceTimeouts: Record> = {}; const SQL_DEBOUNCE_MS = 300; // Handle SQL file changes in the background diff --git a/cli/src/commands/init/init.ts b/cli/src/commands/init/init.ts index c18457464d..471cc858f3 100644 --- a/cli/src/commands/init/init.ts +++ b/cli/src/commands/init/init.ts @@ -231,7 +231,7 @@ async function initAction(opts: InitOptions) { } catch (error) { // If there's an error checking backend settings, just continue with defaults log.warn( - `Could not check backend for git-sync settings: ${error.message}` + `Could not check backend for git-sync settings: ${(error as Error).message}` ); log.info("Continuing with default settings"); } diff --git a/cli/src/commands/schedule/schedule.ts b/cli/src/commands/schedule/schedule.ts index 6eea0de934..bebf37aa9e 100644 --- a/cli/src/commands/schedule/schedule.ts +++ b/cli/src/commands/schedule/schedule.ts @@ -86,7 +86,7 @@ export async function pushSchedule( }); } } catch (e) { - console.error(e.body); + console.error((e as any).body); throw e; } } else { @@ -100,7 +100,7 @@ export async function pushSchedule( }, }); } catch (e) { - console.error(e.body); + console.error((e as any).body); throw e; } } diff --git a/cli/src/commands/script/script.ts b/cli/src/commands/script/script.ts index 7a0b61618d..0bca28c294 100644 --- a/cli/src/commands/script/script.ts +++ b/cli/src/commands/script/script.ts @@ -314,7 +314,7 @@ export async function handleFile( continue; } log.info(`Adding file: ${file.path.substring(1)}`); - const fil = new File([file.contents], file.path.substring(1)); + const fil = new File([file.contents as BlobPart], file.path.substring(1)); tarball.append(fil); } const endTime = performance.now(); @@ -542,7 +542,7 @@ async function streamToBlob(stream: ReadableStream): Promise { } // Create a Blob from the chunks - const blob = new Blob(chunks); + const blob = new Blob(chunks as BlobPart[]); return blob; } diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index 753f871296..e2df1f5679 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -56,7 +56,7 @@ import { isSpecificItem, SpecificItemsConfig, } from "../../core/specific_items.ts"; -import { getCurrentGitBranch } from "../../utils/git.ts"; +import { getCurrentGitBranch, isGitRepository } from "../../utils/git.ts"; import { Workspace } from "../workspace/workspace.ts"; import { removePathPrefix } from "../../types.ts"; import { listSyncCodebases, SyncCodebase } from "../../utils/codebase.ts"; @@ -109,8 +109,9 @@ async function resolveEffectiveSyncOptions( workspace: Workspace, localConfig: SyncOptions, promotion?: string, + branchOverride?: string, ): Promise { - return await getEffectiveSettings(localConfig, promotion); + return await getEffectiveSettings(localConfig, promotion, false, false, branchOverride); } type DynFSElement = { @@ -767,7 +768,7 @@ function ZipFSElement( let inlineScripts; try { inlineScripts = extractInlineScriptsForFlows( - flow.value.modules, + flow.value.modules as any, {}, SEP, defaultTs, @@ -1248,6 +1249,7 @@ export async function elementsToMap( json: boolean, skips: Skips, specificItems?: SpecificItemsConfig, + branchOverride?: string, ): Promise<{ [key: string]: string }> { const map: { [key: string]: string } = {}; const processedBasePaths = new Set(); @@ -1350,8 +1352,7 @@ export async function elementsToMap( // Handle branch-specific files - skip files for other branches if (specificItems && isBranchSpecificFile(path)) { - const currentBranch = getCurrentGitBranch(); - if (!currentBranch || !isCurrentBranchFile(path)) { + if (!isCurrentBranchFile(path, branchOverride)) { // Skip branch-specific files for other branches continue; } @@ -1387,9 +1388,10 @@ export async function elementsToMap( // Handle branch-specific path mapping after all filtering if (specificItems) { - const currentBranch = getCurrentGitBranch(); - if (currentBranch && isCurrentBranchFile(path)) { + if (isCurrentBranchFile(path, branchOverride)) { // This is a branch-specific file for current branch + // Safe to compute branch here since isCurrentBranchFile already validated it exists + const currentBranch = branchOverride || getCurrentGitBranch()!; const basePath = fromBranchSpecificPath(path, currentBranch); if (isSpecificItem(basePath, specificItems)) { // Map to base path for push operations @@ -1444,13 +1446,14 @@ async function compareDynFSElement( codebases: SyncCodebase[], ignoreCodebaseChanges: boolean, specificItems?: SpecificItemsConfig, + branchOverride?: string, ): Promise { const [m1, m2] = els2 ? await Promise.all([ - elementsToMap(els1, ignore, json, skips, specificItems), - elementsToMap(els2, ignore, json, skips, specificItems), + elementsToMap(els1, ignore, json, skips, specificItems, branchOverride), + elementsToMap(els2, ignore, json, skips, specificItems, branchOverride), ]) - : [await elementsToMap(els1, ignore, json, skips, specificItems), {}]; + : [await elementsToMap(els1, ignore, json, skips, specificItems, branchOverride), {}]; const changes: Change[] = []; @@ -1858,7 +1861,7 @@ async function buildTracker(changes: Change[]) { export async function pull( opts: GlobalOptions & - SyncOptions & { repository?: string; promotion?: string }, + SyncOptions & { repository?: string; promotion?: string; branch?: string }, ) { const originalCliOpts = { ...opts }; opts = await mergeConfigWithConfigFile(opts); @@ -1878,7 +1881,7 @@ export async function pull( await ensureDir(path.join(Deno.cwd(), ".wmill")); } - const workspace = await resolveWorkspace(opts); + const workspace = await resolveWorkspace(opts, opts.branch); await requireLogin(opts); // Resolve effective sync options with branch awareness @@ -1886,10 +1889,11 @@ export async function pull( workspace, opts, opts.promotion, + opts.branch, ); // Extract specific items configuration before merging overwrites gitBranches - const specificItems = getSpecificItemsForCurrentBranch(opts); + const specificItems = getSpecificItemsForCurrentBranch(opts, opts.branch); // Merge CLI flags with resolved settings (CLI flags take precedence only for explicit overrides) opts = mergeCliWithEffectiveOptions(originalCliOpts, effectiveOpts); @@ -1947,6 +1951,7 @@ export async function pull( codebases, true, specificItems, + opts.branch, ); log.info( @@ -1969,6 +1974,7 @@ export async function pull( branch_specific_path: getBranchSpecificPath( change.path, specificItems, + opts.branch, ), } : {}), @@ -1981,7 +1987,7 @@ export async function pull( if (changes.length > 0) { if (!opts.jsonOutput) { - prettyChanges(changes, specificItems); + prettyChanges(changes, specificItems, opts.branch); } if (opts.dryRun) { log.info(colors.gray(`Dry run complete.`)); @@ -2007,6 +2013,7 @@ export async function pull( const branchSpecificPath = getBranchSpecificPath( change.path, specificItems, + opts.branch, ); if (branchSpecificPath) { targetPath = branchSpecificPath; @@ -2216,6 +2223,7 @@ export async function pull( branch_specific_path: getBranchSpecificPath( change.path, specificItems, + opts.branch, ), } : {}), @@ -2241,7 +2249,7 @@ export async function pull( } } -function prettyChanges(changes: Change[], specificItems?: SpecificItemsConfig) { +function prettyChanges(changes: Change[], specificItems?: SpecificItemsConfig, branchOverride?: string) { for (const change of changes) { let displayPath = change.path; let branchNote = ""; @@ -2251,6 +2259,7 @@ function prettyChanges(changes: Change[], specificItems?: SpecificItemsConfig) { const branchSpecificPath = getBranchSpecificPath( change.path, specificItems, + branchOverride, ); if (branchSpecificPath) { displayPath = branchSpecificPath; @@ -2334,7 +2343,7 @@ function removeSuffix(str: string, suffix: string) { } export async function push( - opts: GlobalOptions & SyncOptions & { repository?: string }, + opts: GlobalOptions & SyncOptions & { repository?: string; branch?: string }, ) { // Save original CLI options before merging with config file const originalCliOpts = { ...opts }; @@ -2353,7 +2362,7 @@ export async function push( throw error; } - const workspace = await resolveWorkspace(opts); + const workspace = await resolveWorkspace(opts, opts.branch); await requireLogin(opts); // Resolve effective sync options with branch awareness @@ -2361,10 +2370,11 @@ export async function push( workspace, opts, opts.promotion, + opts.branch, ); // Extract specific items configuration BEFORE merging overwrites gitBranches - const specificItems = getSpecificItemsForCurrentBranch(opts); + const specificItems = getSpecificItemsForCurrentBranch(opts, opts.branch); // Merge CLI flags with resolved settings (CLI flags take precedence only for explicit overrides) opts = mergeCliWithEffectiveOptions(originalCliOpts, effectiveOpts); @@ -2432,6 +2442,7 @@ export async function push( codebases, false, specificItems, + opts.branch, ); const rawWorkspaceDependencies = await getRawWorkspaceDependencies(); @@ -2555,6 +2566,7 @@ export async function push( branch_specific_path: getBranchSpecificPath( change.path, specificItems, + opts.branch, ), } : {}), @@ -2567,7 +2579,7 @@ export async function push( if (changes.length > 0) { if (!opts.jsonOutput) { - prettyChanges(changes, specificItems); + prettyChanges(changes, specificItems, opts.branch); } if (opts.dryRun) { log.info(colors.gray(`Dry run complete.`)); @@ -2709,7 +2721,7 @@ export async function push( // For branch-specific resources, push to the base path on the workspace server // This ensures branch-specific files are stored with their base names in the workspace let serverPath = resourceFilePath; - const currentBranch = getCurrentGitBranch(); + const currentBranch = opts.branch || (isGitRepository() ? getCurrentGitBranch() : null); if (currentBranch && isBranchSpecificFile(resourceFilePath)) { serverPath = fromBranchSpecificPath( @@ -2740,6 +2752,7 @@ export async function push( originalBranchSpecificPath = getBranchSpecificPath( change.path, specificItems, + opts.branch, ); } @@ -2793,6 +2806,7 @@ export async function push( const branchSpecificPath = getBranchSpecificPath( change.path, specificItems, + opts.branch, ); if (branchSpecificPath) { localFilePath = branchSpecificPath; @@ -3029,6 +3043,7 @@ export async function push( branch_specific_path: getBranchSpecificPath( change.path, specificItems, + opts.branch, ), } : {}), @@ -3118,6 +3133,10 @@ const command = new Command() "--promotion ", "Use promotionOverrides from the specified branch instead of regular overrides", ) + .option( + "--branch ", + "Override the current git branch (works even outside a git repository)", + ) // deno-lint-ignore no-explicit-any .action(pull as any) .command("push") @@ -3171,6 +3190,10 @@ const command = new Command() "--repository ", "Specify repository path (e.g., u/user/repo) when multiple repositories exist", ) + .option( + "--branch ", + "Override the current git branch (works even outside a git repository)", + ) // deno-lint-ignore no-explicit-any .action(push as any); diff --git a/cli/src/commands/trigger/trigger.ts b/cli/src/commands/trigger/trigger.ts index ecad4ed35e..560acd2f7f 100644 --- a/cli/src/commands/trigger/trigger.ts +++ b/cli/src/commands/trigger/trigger.ts @@ -162,7 +162,7 @@ export async function pushTrigger( path, } as Trigger[K]); } catch (e) { - console.error(e.body); + console.error((e as any).body); throw e; } } else { @@ -175,7 +175,7 @@ export async function pushTrigger( path, } as Trigger[K]); } catch (e) { - console.error(e.body); + console.error((e as any).body); throw e; } } diff --git a/cli/src/commands/workspace/fork.ts b/cli/src/commands/workspace/fork.ts index 5a73021d43..0bd3b9eec6 100644 --- a/cli/src/commands/workspace/fork.ts +++ b/cli/src/commands/workspace/fork.ts @@ -118,7 +118,7 @@ async function createWorkspaceFork( } catch (error) { // If workspace creation fails, we should clean up the git branch log.error( - colors.red(`Failed to create forked workspace: ${error.message}`), + colors.red(`Failed to create forked workspace: ${(error as Error).message}`), ); throw error; } diff --git a/cli/src/commands/workspace/workspace.ts b/cli/src/commands/workspace/workspace.ts index f5878f1251..021e43fea1 100644 --- a/cli/src/commands/workspace/workspace.ts +++ b/cli/src/commands/workspace/workspace.ts @@ -482,7 +482,7 @@ async function bind( try { await Deno.writeTextFile("wmill.yaml", yamlStringify(config)); } catch (error) { - log.error(colors.red(`Failed to save configuration: ${error.message}`)); + log.error(colors.red(`Failed to save configuration: ${(error as Error).message}`)); return; } } diff --git a/cli/src/core/conf.ts b/cli/src/core/conf.ts index d2edc8569c..9de4f8f6e3 100644 --- a/cli/src/core/conf.ts +++ b/cli/src/core/conf.ts @@ -487,73 +487,80 @@ export async function getEffectiveSettings( config: SyncOptions, promotion?: string, skipBranchValidation?: boolean, - suppressLogs?: boolean + suppressLogs?: boolean, + branchOverride?: string ): Promise { // Start with top-level settings from config const { gitBranches, ...topLevelSettings } = config; const effective = { ...topLevelSettings }; - if (isGitRepository()) { - const branch = getCurrentGitBranch(); + // Determine the branch to use: branchOverride takes precedence, then git detection + let currentBranch: string | null = null; + let originalBranchIfForked: string | null = null; + let rawGitBranch: string | null = null; - const originalBranchIfForked = getOriginalBranchForWorkspaceForks(branch); + if (branchOverride) { + currentBranch = branchOverride; + // Note: "Using branch override" is logged in context.ts when resolving workspace + } else if (isGitRepository()) { + rawGitBranch = getCurrentGitBranch(); + originalBranchIfForked = getOriginalBranchForWorkspaceForks(rawGitBranch); - let currentBranch: string | null; if (originalBranchIfForked) { log.info( `Using overrides from original branch \`${originalBranchIfForked}\`` ); currentBranch = originalBranchIfForked; } else { - currentBranch = branch; - } - - // If promotion is specified, use that branch's promotionOverrides or overrides - if (promotion && gitBranches && gitBranches[promotion]) { - const targetBranch = gitBranches[promotion]; - - // First try promotionOverrides, then fall back to overrides - if (targetBranch.promotionOverrides) { - Object.assign(effective, targetBranch.promotionOverrides); - if (!suppressLogs) { - log.info(`Applied promotion settings from branch: ${promotion}`); - } - } else if (targetBranch.overrides) { - Object.assign(effective, targetBranch.overrides); - if (!suppressLogs) { - log.info( - `Applied settings from branch: ${promotion} (no promotionOverrides found)` - ); - } - } else { - log.debug( - `No promotion or regular overrides found for branch '${promotion}', using top-level settings` - ); - } - } - // Otherwise use current branch overrides (existing behavior) - else if ( - currentBranch && - gitBranches && - gitBranches[currentBranch] && - gitBranches[currentBranch].overrides - ) { - Object.assign(effective, gitBranches[currentBranch].overrides); - if (!suppressLogs) { - const extraLog = originalBranchIfForked - ? ` (because it is the origin of the workspace fork branch \`${branch}\`)` - : ""; - log.info( - `Applied settings for Git branch: ${currentBranch}${extraLog}` - ); - } - } else if (currentBranch) { - log.debug( - `No branch-specific overrides found for '${currentBranch}', using top-level settings` - ); + currentBranch = rawGitBranch; } } else { - log.debug("Not in a Git repository, using top-level settings"); + log.debug("Not in a Git repository and no branch override provided, using top-level settings"); + } + + // If promotion is specified, use that branch's promotionOverrides or overrides + if (promotion && gitBranches && gitBranches[promotion]) { + const targetBranch = gitBranches[promotion]; + + // First try promotionOverrides, then fall back to overrides + if (targetBranch.promotionOverrides) { + Object.assign(effective, targetBranch.promotionOverrides); + if (!suppressLogs) { + log.info(`Applied promotion settings from branch: ${promotion}`); + } + } else if (targetBranch.overrides) { + Object.assign(effective, targetBranch.overrides); + if (!suppressLogs) { + log.info( + `Applied settings from branch: ${promotion} (no promotionOverrides found)` + ); + } + } else { + log.debug( + `No promotion or regular overrides found for branch '${promotion}', using top-level settings` + ); + } + } + // Otherwise use current branch overrides (existing behavior) + else if ( + currentBranch && + gitBranches && + gitBranches[currentBranch] && + gitBranches[currentBranch].overrides + ) { + Object.assign(effective, gitBranches[currentBranch].overrides); + if (!suppressLogs) { + const extraLog = originalBranchIfForked + ? ` (because it is the origin of the workspace fork branch \`${rawGitBranch}\`)` + : ""; + log.info( + `Applied settings for Git branch: ${currentBranch}${extraLog}` + ); + } + } else if (currentBranch) { + log.debug( + `No branch-specific overrides found for '${currentBranch}', using top-level settings` + ); } return effective; diff --git a/cli/src/core/context.ts b/cli/src/core/context.ts index ad8f34a30e..714b613cdd 100644 --- a/cli/src/core/context.ts +++ b/cli/src/core/context.ts @@ -221,29 +221,40 @@ async function tryResolveWorkspace( } export async function tryResolveBranchWorkspace( - opts: GlobalOptions + opts: GlobalOptions, + branchOverride?: string ): Promise { - // Only try branch-based resolution if in a Git repository - if (!isGitRepository()) { - return undefined; - } - - const rawBranch = getCurrentGitBranch(); - if (!rawBranch) { - return undefined; - } - + let rawBranch: string | null = null; let currentBranch: string; - const originalBranchIfForked = getOriginalBranchForWorkspaceForks(rawBranch); - const workspaceIdIfForked = - getWorkspaceIdForWorkspaceForkFromBranchName(rawBranch); - if (originalBranchIfForked) { - log.info( - `Using original branch \`${originalBranchIfForked}\` for finding workspace profile from gitBranches section in wmill.yaml` - ); - currentBranch = originalBranchIfForked; + let originalBranchIfForked: string | null = null; + let workspaceIdIfForked: string | null = null; + + if (branchOverride) { + // Use branch override directly + currentBranch = branchOverride; + log.info(`Using branch override: ${branchOverride}`); } else { - currentBranch = rawBranch; + // Only try branch-based resolution if in a Git repository + if (!isGitRepository()) { + return undefined; + } + + rawBranch = getCurrentGitBranch(); + if (!rawBranch) { + return undefined; + } + + originalBranchIfForked = getOriginalBranchForWorkspaceForks(rawBranch); + workspaceIdIfForked = + getWorkspaceIdForWorkspaceForkFromBranchName(rawBranch); + if (originalBranchIfForked) { + log.info( + `Using original branch \`${originalBranchIfForked}\` for finding workspace profile from gitBranches section in wmill.yaml` + ); + currentBranch = originalBranchIfForked; + } else { + currentBranch = rawBranch; + } } // Read wmill.yaml to check for branch workspace configuration @@ -284,7 +295,7 @@ export async function tryResolveBranchWorkspace( workspaceId, currentBranch, opts, - { rawBranch, isForked: !!originalBranchIfForked } + { rawBranch: rawBranch ?? currentBranch, isForked: !!originalBranchIfForked } ); } @@ -358,7 +369,8 @@ export async function tryResolveBranchWorkspace( } export async function resolveWorkspace( - opts: GlobalOptions + opts: GlobalOptions, + branchOverride?: string ): Promise { const cache = (opts as any).__secret_workspace; if (cache) return cache; @@ -438,13 +450,13 @@ export async function resolveWorkspace( } } - const branch = getCurrentGitBranch(); + const branch = branchOverride ?? getCurrentGitBranch(); // Try explicit workspace flag first (should override branch-based resolution). Unless it's a - // forked workspace, that we detect through the branch name + // forked workspace, that we detect through the branch name (only when not using branchOverride) const res = await tryResolveWorkspace(opts); if (!res.isError) { - if (!branch || !branch.startsWith(WM_FORK_PREFIX)) { + if (branchOverride || !branch || !branch.startsWith(WM_FORK_PREFIX)) { return res.value; } else { log.info( @@ -454,11 +466,12 @@ export async function resolveWorkspace( } // Try branch-based resolution (medium priority) - const branchWorkspace = await tryResolveBranchWorkspace(opts); + const branchWorkspace = await tryResolveBranchWorkspace(opts, branchOverride); if (branchWorkspace) { (opts as any).__secret_workspace = branchWorkspace; return branchWorkspace; - } else { + } else if (!branchOverride) { + // Only check for fork errors when not using branchOverride const originalBranch = getOriginalBranchForWorkspaceForks(branch); if (originalBranch) { log.error( diff --git a/cli/src/core/specific_items.ts b/cli/src/core/specific_items.ts index 2e9b5052f0..bef871b598 100644 --- a/cli/src/core/specific_items.ts +++ b/cli/src/core/specific_items.ts @@ -60,12 +60,19 @@ function buildYamlTypePattern(): string { * Get the specific items configuration for the current git branch * Merges commonSpecificItems with branch-specific specificItems */ -export function getSpecificItemsForCurrentBranch(config: SyncOptions): SpecificItemsConfig | undefined { - if (!isGitRepository() || !config.gitBranches) { +export function getSpecificItemsForCurrentBranch(config: SyncOptions, branchOverride?: string): SpecificItemsConfig | undefined { + if (!config.gitBranches) { return undefined; } - const currentBranch = getCurrentGitBranch(); + // Use branch override if provided, otherwise detect from git + let currentBranch: string | null = null; + if (branchOverride) { + currentBranch = branchOverride; + } else if (isGitRepository()) { + currentBranch = getCurrentGitBranch(); + } + if (!currentBranch) { return undefined; } @@ -224,13 +231,21 @@ export function fromBranchSpecificPath(branchSpecificPath: string, branchName: s */ export function getBranchSpecificPath( basePath: string, - specificItems: SpecificItemsConfig | undefined + specificItems: SpecificItemsConfig | undefined, + branchOverride?: string ): string | undefined { - if (!isGitRepository() || !specificItems) { + if (!specificItems) { return undefined; } - const currentBranch = getCurrentGitBranch(); + // Use branch override if provided, otherwise detect from git + let currentBranch: string | null = null; + if (branchOverride) { + currentBranch = branchOverride; + } else if (isGitRepository()) { + currentBranch = getCurrentGitBranch(); + } + if (!currentBranch) { return undefined; } @@ -248,12 +263,15 @@ const branchPatternCache = new Map(); /** * Check if a path is a branch-specific file for the current branch */ -export function isCurrentBranchFile(path: string): boolean { - if (!isGitRepository()) { - return false; +export function isCurrentBranchFile(path: string, branchOverride?: string): boolean { + // Use branch override if provided, otherwise detect from git + let currentBranch: string | null = null; + if (branchOverride) { + currentBranch = branchOverride; + } else if (isGitRepository()) { + currentBranch = getCurrentGitBranch(); } - const currentBranch = getCurrentGitBranch(); if (!currentBranch) { return false; } diff --git a/cli/src/utils/resource_folders.ts b/cli/src/utils/resource_folders.ts index 96beb91635..b939e15be2 100644 --- a/cli/src/utils/resource_folders.ts +++ b/cli/src/utils/resource_folders.ts @@ -34,6 +34,7 @@ export type FolderSuffixes = // Global state for nonDottedPaths configuration let _nonDottedPaths = false; +let _nonDottedPathsLogged = false; /** * Set whether to use non-dotted paths (__flow, __app, __raw_app) @@ -41,8 +42,9 @@ let _nonDottedPaths = false; * This should be called once at startup based on wmill.yaml configuration. */ export function setNonDottedPaths(value: boolean): void { - if (value) { + if (value && !_nonDottedPathsLogged) { log.info("Using non-dotted paths (__flow, __app, __raw_app)"); + _nonDottedPathsLogged = true; } _nonDottedPaths = value; } diff --git a/cli/test/conf_branch_override.test.ts b/cli/test/conf_branch_override.test.ts new file mode 100644 index 0000000000..e0976710bb --- /dev/null +++ b/cli/test/conf_branch_override.test.ts @@ -0,0 +1,144 @@ +import { assertEquals, assertExists } from "https://deno.land/std@0.224.0/assert/mod.ts"; +import { getEffectiveSettings, type SyncOptions } from "../src/core/conf.ts"; + +// ============================================================================= +// CONF.TS BRANCH OVERRIDE TESTS +// Tests for getEffectiveSettings with branchOverride parameter +// ============================================================================= + +Deno.test("getEffectiveSettings: applies branch overrides when branchOverride is provided", async () => { + const config: SyncOptions = { + defaultTs: "bun", + includes: ["f/**"], + gitBranches: { + staging: { + overrides: { + includes: ["staging/**"], + skipVariables: true, + }, + }, + production: { + overrides: { + includes: ["prod/**"], + skipSecrets: true, + }, + }, + }, + }; + + // Test with staging branch override + const stagingSettings = await getEffectiveSettings(config, undefined, true, true, "staging"); + assertEquals(stagingSettings.includes, ["staging/**"]); + assertEquals(stagingSettings.skipVariables, true); + assertEquals(stagingSettings.skipSecrets, undefined); + + // Test with production branch override + const prodSettings = await getEffectiveSettings(config, undefined, true, true, "production"); + assertEquals(prodSettings.includes, ["prod/**"]); + assertEquals(prodSettings.skipSecrets, true); + assertEquals(prodSettings.skipVariables, undefined); +}); + +Deno.test("getEffectiveSettings: uses top-level settings when branchOverride has no overrides", async () => { + const config: SyncOptions = { + defaultTs: "bun", + includes: ["f/**"], + skipVariables: true, + gitBranches: { + staging: { + // No overrides defined + }, + }, + }; + + const settings = await getEffectiveSettings(config, undefined, true, true, "staging"); + assertEquals(settings.includes, ["f/**"]); + assertEquals(settings.skipVariables, true); + assertEquals(settings.defaultTs, "bun"); +}); + +Deno.test("getEffectiveSettings: uses top-level settings for unknown branch", async () => { + const config: SyncOptions = { + defaultTs: "bun", + includes: ["f/**"], + gitBranches: { + staging: { + overrides: { + includes: ["staging/**"], + }, + }, + }, + }; + + const settings = await getEffectiveSettings(config, undefined, true, true, "nonexistent"); + assertEquals(settings.includes, ["f/**"]); + assertEquals(settings.defaultTs, "bun"); +}); + +Deno.test("getEffectiveSettings: promotionOverrides take precedence when promotion specified", async () => { + const config: SyncOptions = { + defaultTs: "bun", + includes: ["f/**"], + gitBranches: { + production: { + overrides: { + includes: ["prod/**"], + }, + promotionOverrides: { + includes: ["promoted/**"], + skipVariables: true, + }, + }, + }, + }; + + // Test without promotion flag - should use regular overrides + const normalSettings = await getEffectiveSettings(config, undefined, true, true, "production"); + assertEquals(normalSettings.includes, ["prod/**"]); + assertEquals(normalSettings.skipVariables, undefined); + + // Test with promotion flag - should use promotionOverrides + const promoSettings = await getEffectiveSettings(config, "production", true, true); + assertEquals(promoSettings.includes, ["promoted/**"]); + assertEquals(promoSettings.skipVariables, true); +}); + +Deno.test("getEffectiveSettings: branchOverride works without gitBranches config", async () => { + const config: SyncOptions = { + defaultTs: "bun", + includes: ["f/**"], + }; + + // Should not throw even with branchOverride but no gitBranches + const settings = await getEffectiveSettings(config, undefined, true, true, "staging"); + assertEquals(settings.includes, ["f/**"]); + assertEquals(settings.defaultTs, "bun"); +}); + +Deno.test("getEffectiveSettings: preserves all top-level settings in merged result", async () => { + const config: SyncOptions = { + defaultTs: "bun", + includes: ["f/**"], + excludes: ["*.test.ts"], + skipVariables: false, + skipResources: false, + skipFlows: false, + parallel: 4, + gitBranches: { + staging: { + overrides: { + skipVariables: true, // Override just this one + }, + }, + }, + }; + + const settings = await getEffectiveSettings(config, undefined, true, true, "staging"); + assertEquals(settings.defaultTs, "bun"); + assertEquals(settings.includes, ["f/**"]); + assertEquals(settings.excludes, ["*.test.ts"]); + assertEquals(settings.skipVariables, true); // Overridden + assertEquals(settings.skipResources, false); + assertEquals(settings.skipFlows, false); + assertEquals(settings.parallel, 4); +}); diff --git a/cli/test/containerized_backend.ts b/cli/test/containerized_backend.ts index b6facf945d..8ee7e064f6 100644 --- a/cli/test/containerized_backend.ts +++ b/cli/test/containerized_backend.ts @@ -20,7 +20,7 @@ export class ContainerizedBackend { constructor(config: Partial = {}) { this.config = { - composeFile: config.composeFile || '/home/alex/windmill/windmill/cli/test/docker-compose.test.yml', + composeFile: config.composeFile || new URL('./docker-compose.test.yml', import.meta.url).pathname, baseUrl: config.baseUrl || 'http://localhost:8001', workspace: config.workspace || 'test', // Use test workspace token: config.token || '', @@ -1023,10 +1023,13 @@ export async function main( ...args ]; - console.log('🔧 CLI Command:', ['/home/alex/.deno/bin/deno', 'run', '-A', '/home/alex/windmill/windmill/cli/src/main.ts', ...fullArgs].join(' ')); - - return new Deno.Command('/home/alex/.deno/bin/deno', { - args: ['run', '-A', '/home/alex/windmill/windmill/cli/src/main.ts', ...fullArgs], + const denoPath = Deno.execPath(); + const cliMainPath = new URL('../src/main.ts', import.meta.url).pathname; + + console.log('🔧 CLI Command:', [denoPath, 'run', '-A', cliMainPath, ...fullArgs].join(' ')); + + return new Deno.Command(denoPath, { + args: ['run', '-A', cliMainPath, ...fullArgs], cwd: workingDir, stdout: 'piped', stderr: 'piped', diff --git a/cli/test/specific_items.test.ts b/cli/test/specific_items.test.ts new file mode 100644 index 0000000000..4448f32db7 --- /dev/null +++ b/cli/test/specific_items.test.ts @@ -0,0 +1,338 @@ +import { assertEquals, assertExists, assert } from "https://deno.land/std@0.224.0/assert/mod.ts"; + +// ============================================================================= +// SPECIFIC ITEMS UNIT TESTS +// Tests for branch-specific file path functions (no Docker required) +// ============================================================================= + +// Import the functions we need to test +import { + isSpecificItem, + toBranchSpecificPath, + fromBranchSpecificPath, + isBranchSpecificFile, +} from "../src/core/specific_items.ts"; + +import type { SpecificItemsConfig } from "../src/core/specific_items.ts"; + +// ============================================================================= +// toBranchSpecificPath TESTS +// ============================================================================= + +Deno.test("toBranchSpecificPath: converts variable path to branch-specific", () => { + const result = toBranchSpecificPath("f/test.variable.yaml", "main"); + assertEquals(result, "f/test.main.variable.yaml"); +}); + +Deno.test("toBranchSpecificPath: converts resource path to branch-specific", () => { + const result = toBranchSpecificPath("u/admin/db.resource.yaml", "develop"); + assertEquals(result, "u/admin/db.develop.resource.yaml"); +}); + +Deno.test("toBranchSpecificPath: converts trigger path to branch-specific", () => { + const result = toBranchSpecificPath("f/my_trigger.http_trigger.yaml", "feature-x"); + assertEquals(result, "f/my_trigger.feature-x.http_trigger.yaml"); +}); + +Deno.test("toBranchSpecificPath: sanitizes branch names with slashes", () => { + const result = toBranchSpecificPath("f/test.variable.yaml", "feature/my-feature"); + assertEquals(result, "f/test.feature_my-feature.variable.yaml"); +}); + +Deno.test("toBranchSpecificPath: sanitizes branch names with dots", () => { + const result = toBranchSpecificPath("f/test.variable.yaml", "release.1.0"); + assertEquals(result, "f/test.release_1_0.variable.yaml"); +}); + +Deno.test("toBranchSpecificPath: leaves non-specific files unchanged", () => { + const result = toBranchSpecificPath("f/script.ts", "main"); + assertEquals(result, "f/script.ts"); +}); + +Deno.test("toBranchSpecificPath: handles resource files with extensions", () => { + const result = toBranchSpecificPath("f/config.resource.file.json", "main"); + assertEquals(result, "f/config.main.resource.file.json"); +}); + +// ============================================================================= +// fromBranchSpecificPath TESTS +// ============================================================================= + +Deno.test("fromBranchSpecificPath: converts branch-specific variable back to base", () => { + const result = fromBranchSpecificPath("f/test.main.variable.yaml", "main"); + assertEquals(result, "f/test.variable.yaml"); +}); + +Deno.test("fromBranchSpecificPath: converts branch-specific resource back to base", () => { + const result = fromBranchSpecificPath("u/admin/db.develop.resource.yaml", "develop"); + assertEquals(result, "u/admin/db.resource.yaml"); +}); + +Deno.test("fromBranchSpecificPath: converts branch-specific trigger back to base", () => { + const result = fromBranchSpecificPath("f/my_trigger.feature-x.http_trigger.yaml", "feature-x"); + assertEquals(result, "f/my_trigger.http_trigger.yaml"); +}); + +Deno.test("fromBranchSpecificPath: handles sanitized branch names", () => { + const result = fromBranchSpecificPath("f/test.feature_my-feature.variable.yaml", "feature/my-feature"); + assertEquals(result, "f/test.variable.yaml"); +}); + +Deno.test("fromBranchSpecificPath: returns unchanged if not branch-specific", () => { + const result = fromBranchSpecificPath("f/test.variable.yaml", "main"); + assertEquals(result, "f/test.variable.yaml"); +}); + +Deno.test("fromBranchSpecificPath: handles resource files with extensions", () => { + const result = fromBranchSpecificPath("f/config.main.resource.file.json", "main"); + assertEquals(result, "f/config.resource.file.json"); +}); + +// ============================================================================= +// isSpecificItem TESTS +// ============================================================================= + +Deno.test("isSpecificItem: returns false when specificItems is undefined", () => { + const result = isSpecificItem("f/test.variable.yaml", undefined); + assertEquals(result, false); +}); + +Deno.test("isSpecificItem: matches variable paths with glob pattern", () => { + const config: SpecificItemsConfig = { + variables: ["f/**"], + }; + assertEquals(isSpecificItem("f/test.variable.yaml", config), true); + assertEquals(isSpecificItem("u/admin/test.variable.yaml", config), false); +}); + +Deno.test("isSpecificItem: matches resource paths with glob pattern", () => { + const config: SpecificItemsConfig = { + resources: ["u/admin/**"], + }; + assertEquals(isSpecificItem("u/admin/db.resource.yaml", config), true); + assertEquals(isSpecificItem("f/db.resource.yaml", config), false); +}); + +Deno.test("isSpecificItem: matches trigger paths with glob pattern", () => { + const config: SpecificItemsConfig = { + triggers: ["f/triggers/**"], + }; + assertEquals(isSpecificItem("f/triggers/my.http_trigger.yaml", config), true); + assertEquals(isSpecificItem("u/admin/my.http_trigger.yaml", config), false); +}); + +Deno.test("isSpecificItem: matches multiple patterns", () => { + const config: SpecificItemsConfig = { + variables: ["f/**", "g/**"], + }; + assertEquals(isSpecificItem("f/test.variable.yaml", config), true); + assertEquals(isSpecificItem("g/test.variable.yaml", config), true); + assertEquals(isSpecificItem("u/admin/test.variable.yaml", config), false); +}); + +Deno.test("isSpecificItem: handles exact path patterns", () => { + const config: SpecificItemsConfig = { + variables: ["f/specific.variable.yaml"], + }; + assertEquals(isSpecificItem("f/specific.variable.yaml", config), true); + assertEquals(isSpecificItem("f/other.variable.yaml", config), false); +}); + +// ============================================================================= +// isBranchSpecificFile TESTS +// ============================================================================= + +Deno.test("isBranchSpecificFile: detects branch-specific variable files", () => { + assertEquals(isBranchSpecificFile("f/test.main.variable.yaml"), true); + assertEquals(isBranchSpecificFile("f/test.develop.variable.yaml"), true); + assertEquals(isBranchSpecificFile("f/test.feature_branch.variable.yaml"), true); +}); + +Deno.test("isBranchSpecificFile: detects branch-specific resource files", () => { + assertEquals(isBranchSpecificFile("u/admin/db.main.resource.yaml"), true); + assertEquals(isBranchSpecificFile("u/admin/db.staging.resource.yaml"), true); +}); + +Deno.test("isBranchSpecificFile: detects branch-specific trigger files", () => { + assertEquals(isBranchSpecificFile("f/my.main.http_trigger.yaml"), true); + assertEquals(isBranchSpecificFile("f/my.develop.kafka_trigger.yaml"), true); + assertEquals(isBranchSpecificFile("f/my.main.websocket_trigger.yaml"), true); +}); + +Deno.test("isBranchSpecificFile: returns false for non-branch-specific files", () => { + assertEquals(isBranchSpecificFile("f/test.variable.yaml"), false); + assertEquals(isBranchSpecificFile("u/admin/db.resource.yaml"), false); + assertEquals(isBranchSpecificFile("f/my.http_trigger.yaml"), false); + assertEquals(isBranchSpecificFile("f/script.ts"), false); +}); + +Deno.test("isBranchSpecificFile: handles resource files with extensions", () => { + assertEquals(isBranchSpecificFile("f/config.main.resource.file.json"), true); + assertEquals(isBranchSpecificFile("f/config.resource.file.json"), false); +}); + +// ============================================================================= +// ROUND-TRIP TESTS +// ============================================================================= + +Deno.test("round-trip: variable file path conversion", () => { + const original = "f/my/nested/config.variable.yaml"; + const branch = "feature/test-branch"; + const branchSpecific = toBranchSpecificPath(original, branch); + const restored = fromBranchSpecificPath(branchSpecific, branch); + assertEquals(restored, original); +}); + +Deno.test("round-trip: resource file path conversion", () => { + const original = "u/admin/database.resource.yaml"; + const branch = "develop"; + const branchSpecific = toBranchSpecificPath(original, branch); + const restored = fromBranchSpecificPath(branchSpecific, branch); + assertEquals(restored, original); +}); + +Deno.test("round-trip: trigger file path conversion", () => { + const original = "f/webhooks/handler.http_trigger.yaml"; + const branch = "main"; + const branchSpecific = toBranchSpecificPath(original, branch); + const restored = fromBranchSpecificPath(branchSpecific, branch); + assertEquals(restored, original); +}); + +Deno.test("round-trip: resource file with extension", () => { + const original = "f/configs/settings.resource.file.ini"; + const branch = "release/v1.0"; + const branchSpecific = toBranchSpecificPath(original, branch); + const restored = fromBranchSpecificPath(branchSpecific, branch); + assertEquals(restored, original); +}); + +// ============================================================================= +// BRANCH OVERRIDE TESTS (for --branch flag functionality) +// These tests validate that functions work correctly with explicit branch override +// ============================================================================= + +import { + getBranchSpecificPath, + isCurrentBranchFile, + getSpecificItemsForCurrentBranch, +} from "../src/core/specific_items.ts"; + +Deno.test("branchOverride: getBranchSpecificPath with override returns branch-specific path", () => { + // This test verifies that when branchOverride is provided, the function uses it + // instead of detecting the current git branch + const config: SpecificItemsConfig = { + variables: ["f/**"], + }; + + // When override is provided, it should return the branch-specific path even outside git repo + const result = getBranchSpecificPath("f/test.variable.yaml", config, "staging"); + assertEquals(result, "f/test.staging.variable.yaml"); +}); + +Deno.test("branchOverride: getBranchSpecificPath without override and not in git repo returns undefined", () => { + const config: SpecificItemsConfig = { + variables: ["f/**"], + }; + + // Without override and outside git repo (or if git returns null), should return undefined + // Note: This test's behavior depends on whether we're in a git repo + const result = getBranchSpecificPath("f/test.variable.yaml", config); + // In a git repo, this would return a branch-specific path; outside, it would be undefined + // We test the override case above which is deterministic +}); + +Deno.test("branchOverride: isCurrentBranchFile with override uses provided branch", () => { + // Test that isCurrentBranchFile uses the override branch instead of git detection + const result = isCurrentBranchFile("f/test.staging.variable.yaml", "staging"); + assertEquals(result, true); + + // Should return false for different branch + const resultOther = isCurrentBranchFile("f/test.staging.variable.yaml", "production"); + assertEquals(resultOther, false); + + // Should return false for non-branch-specific file + const resultNonSpecific = isCurrentBranchFile("f/test.variable.yaml", "staging"); + assertEquals(resultNonSpecific, false); +}); + +Deno.test("branchOverride: isCurrentBranchFile with override handles sanitized branch names", () => { + // Test with branch names that get sanitized + const result = isCurrentBranchFile("f/test.feature_my-branch.variable.yaml", "feature/my-branch"); + assertEquals(result, true); + + // Different sanitized branch should return false + const resultOther = isCurrentBranchFile("f/test.feature_my-branch.variable.yaml", "feature/other-branch"); + assertEquals(resultOther, false); +}); + +Deno.test("branchOverride: getSpecificItemsForCurrentBranch with override returns correct config", () => { + // Test that getSpecificItemsForCurrentBranch uses the override branch + const config = { + gitBranches: { + staging: { + specificItems: { + variables: ["f/**"], + resources: ["u/admin/**"], + }, + }, + production: { + specificItems: { + variables: ["g/**"], + }, + }, + commonSpecificItems: { + triggers: ["f/webhooks/**"], + }, + }, + }; + + const stagingItems = getSpecificItemsForCurrentBranch(config as any, "staging"); + assertEquals(stagingItems?.variables, ["f/**"]); + assertEquals(stagingItems?.resources, ["u/admin/**"]); + assertEquals(stagingItems?.triggers, ["f/webhooks/**"]); // From common + + const productionItems = getSpecificItemsForCurrentBranch(config as any, "production"); + assertEquals(productionItems?.variables, ["g/**"]); + assertEquals(productionItems?.resources, undefined); + assertEquals(productionItems?.triggers, ["f/webhooks/**"]); // From common +}); + +Deno.test("branchOverride: getSpecificItemsForCurrentBranch with non-existent branch returns undefined", () => { + const config = { + gitBranches: { + staging: { + specificItems: { + variables: ["f/**"], + }, + }, + }, + }; + + // When the branch doesn't have specific items (and there's no common), should return undefined + const result = getSpecificItemsForCurrentBranch(config as any, "nonexistent"); + assertEquals(result, undefined); +}); + +Deno.test("branchOverride: getSpecificItemsForCurrentBranch merges common and branch items", () => { + const config = { + gitBranches: { + commonSpecificItems: { + variables: ["common/**"], + resources: ["shared/**"], + }, + develop: { + specificItems: { + variables: ["dev/**"], + triggers: ["dev/triggers/**"], + }, + }, + }, + }; + + const result = getSpecificItemsForCurrentBranch(config as any, "develop"); + // Should merge common and branch-specific + assertEquals(result?.variables, ["common/**", "dev/**"]); + assertEquals(result?.resources, ["shared/**"]); + assertEquals(result?.triggers, ["dev/triggers/**"]); +}); From 437bad4cb7d6bfac136b1b41971edba66cd93eb9 Mon Sep 17 00:00:00 2001 From: centdix <40307056+centdix@users.noreply.github.com> Date: Fri, 16 Jan 2026 21:55:14 +0100 Subject: [PATCH 02/19] chore: move mcp logic to windmill-mcp (#7584) * draft * clean up mcp logic * cleaning * cleaning * better code * error logging * cleaning --- backend/Cargo.lock | 6 + .../generate_mcp_tools.py | 28 +- backend/windmill-api/src/lib.rs | 5 +- .../{tools => }/auto_generated_endpoints.rs | 16 +- backend/windmill-api/src/mcp/core.rs | 448 ++++++++++++++ backend/windmill-api/src/mcp/mod.rs | 10 +- backend/windmill-api/src/mcp/server.rs | 561 ------------------ .../src/mcp/tools/endpoint_tools.rs | 307 ---------- .../windmill-api/src/mcp/tools/flow_tools.rs | 40 -- .../windmill-api/src/mcp/tools/hub_tools.rs | 43 -- backend/windmill-api/src/mcp/tools/mod.rs | 10 - .../src/mcp/tools/script_tools.rs | 40 -- backend/windmill-api/src/mcp/utils.rs | 414 +++++++++++++ .../windmill-api/src/mcp/utils/database.rs | 243 -------- backend/windmill-api/src/mcp/utils/mod.rs | 10 - backend/windmill-api/src/mcp/utils/schema.rs | 143 ----- backend/windmill-mcp/Cargo.toml | 8 +- backend/windmill-mcp/src/client/mod.rs | 209 +++++++ backend/windmill-mcp/src/client/types.rs | 32 + backend/windmill-mcp/src/common/mod.rs | 16 + backend/windmill-mcp/src/common/schema.rs | 41 ++ .../src/common/scope.rs} | 48 +- .../src/common}/transform.rs | 39 +- .../src/common/types.rs} | 37 +- backend/windmill-mcp/src/lib.rs | 275 +-------- backend/windmill-mcp/src/server/backend.rs | 159 +++++ backend/windmill-mcp/src/server/endpoints.rs | 103 ++++ backend/windmill-mcp/src/server/mod.rs | 32 + backend/windmill-mcp/src/server/runner.rs | 373 ++++++++++++ backend/windmill-mcp/src/server/tools.rs | 184 ++++++ 30 files changed, 2161 insertions(+), 1719 deletions(-) rename backend/windmill-api/src/mcp/{tools => }/auto_generated_endpoints.rs (99%) create mode 100644 backend/windmill-api/src/mcp/core.rs delete mode 100644 backend/windmill-api/src/mcp/server.rs delete mode 100644 backend/windmill-api/src/mcp/tools/endpoint_tools.rs delete mode 100644 backend/windmill-api/src/mcp/tools/flow_tools.rs delete mode 100644 backend/windmill-api/src/mcp/tools/hub_tools.rs delete mode 100644 backend/windmill-api/src/mcp/tools/mod.rs delete mode 100644 backend/windmill-api/src/mcp/tools/script_tools.rs create mode 100644 backend/windmill-api/src/mcp/utils.rs delete mode 100644 backend/windmill-api/src/mcp/utils/database.rs delete mode 100644 backend/windmill-api/src/mcp/utils/mod.rs delete mode 100644 backend/windmill-api/src/mcp/utils/schema.rs create mode 100644 backend/windmill-mcp/src/client/mod.rs create mode 100644 backend/windmill-mcp/src/client/types.rs create mode 100644 backend/windmill-mcp/src/common/mod.rs create mode 100644 backend/windmill-mcp/src/common/schema.rs rename backend/{windmill-api/src/mcp/utils/scope_matcher.rs => windmill-mcp/src/common/scope.rs} (86%) rename backend/{windmill-api/src/mcp/utils => windmill-mcp/src/common}/transform.rs (78%) rename backend/{windmill-api/src/mcp/utils/models.rs => windmill-mcp/src/common/types.rs} (68%) create mode 100644 backend/windmill-mcp/src/server/backend.rs create mode 100644 backend/windmill-mcp/src/server/endpoints.rs create mode 100644 backend/windmill-mcp/src/server/mod.rs create mode 100644 backend/windmill-mcp/src/server/runner.rs create mode 100644 backend/windmill-mcp/src/server/tools.rs diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 7455c4e5a6..af26999421 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -15624,11 +15624,17 @@ name = "windmill-mcp" version = "1.608.0" dependencies = [ "anyhow", + "async-trait", + "futures", + "http 1.4.0", "oauth2", "reqwest 0.12.28", "rmcp", "serde", "serde_json", + "sqlx", + "tokio", + "tokio-util", "tracing", "windmill-common", ] diff --git a/backend/generate_mcp_endpoints_tools/generate_mcp_tools.py b/backend/generate_mcp_endpoints_tools/generate_mcp_tools.py index ae1e27a288..154e3e4376 100644 --- a/backend/generate_mcp_endpoints_tools/generate_mcp_tools.py +++ b/backend/generate_mcp_endpoints_tools/generate_mcp_tools.py @@ -13,21 +13,7 @@ from typing import Dict, List, Any, Optional IMPORTS = """ use std::borrow::Cow; -use serde::{Deserialize, Serialize}; -""" - -ENDPOINT_STRUCT = """ -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct EndpointTool { - pub name: Cow<'static, str>, - pub description: Cow<'static, str>, - pub instructions: Cow<'static, str>, - pub path: Cow<'static, str>, - pub method: Cow<'static, str>, - pub path_params_schema: Option, - pub query_params_schema: Option, - pub body_schema: Option, -} +use windmill_mcp::server::EndpointTool; """ def load_openapi_spec(file_path: str) -> Dict[str, Any]: @@ -341,13 +327,11 @@ export const mcpEndpointTools: EndpointTool[] = [ def generate_rust_code(tools: List[Dict[str, Any]], spec: Dict[str, Any], base_path: str = "") -> str: """Generate the complete Rust code with MCP tools.""" if not tools: - return """// No MCP tools found in the OpenAPI specification - + return f"""// No MCP tools found in the OpenAPI specification {IMPORTS} -{ENDPOINT_STRUCT} -pub fn all_tools() -> Vec { +pub fn all_tools() -> Vec {{ vec![] -} +}} """ tool_definitions = [] @@ -386,9 +370,7 @@ pub fn all_tools() -> Vec { rust_code = f"""// Auto-generated MCP tools from OpenAPI specification // This file is generated by generate_mcp_tools.py - DO NOT EDIT MANUALLY - {IMPORTS} -{ENDPOINT_STRUCT} pub fn all_tools() -> Vec {{ vec![ {tool_definitions_str} @@ -405,7 +387,7 @@ def main(): project_dir = backend_dir.parent openapi_file = backend_dir / "windmill-api" / "openapi.yaml" - rust_output_file = backend_dir / "windmill-api" / "src" / "mcp" / "tools" / "auto_generated_endpoints.rs" + rust_output_file = backend_dir / "windmill-api" / "src" / "mcp" / "auto_generated_endpoints.rs" ts_output_file = project_dir / "frontend" / "src" / "lib" / "mcpEndpointTools.ts" if not openapi_file.exists(): diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index a46f37a811..df00619425 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -355,7 +355,7 @@ pub async fn run_server( { let smtp_server = Arc::new(SmtpServer { db: db.clone(), - user_db: user_db, + user_db: user_db.clone(), auth_cache: auth_cache.clone(), base_internal_url: _base_internal_url.clone(), }); @@ -409,7 +409,8 @@ pub async fn run_server( let (mcp_router, mcp_cancellation_token) = { #[cfg(feature = "mcp")] if server_mode || mcp_mode { - let (mcp_router, mcp_cancellation_token) = setup_mcp_server().await?; + let (mcp_router, mcp_cancellation_token) = + setup_mcp_server(db.clone(), user_db).await?; let mcp_middleware = axum::middleware::from_fn(extract_and_store_workspace_id); ( mcp_router.layer(mcp_middleware), diff --git a/backend/windmill-api/src/mcp/tools/auto_generated_endpoints.rs b/backend/windmill-api/src/mcp/auto_generated_endpoints.rs similarity index 99% rename from backend/windmill-api/src/mcp/tools/auto_generated_endpoints.rs rename to backend/windmill-api/src/mcp/auto_generated_endpoints.rs index d5425dcd7f..91a07e3f7a 100644 --- a/backend/windmill-api/src/mcp/tools/auto_generated_endpoints.rs +++ b/backend/windmill-api/src/mcp/auto_generated_endpoints.rs @@ -1,22 +1,8 @@ // Auto-generated MCP tools from OpenAPI specification // This file is generated by generate_mcp_tools.py - DO NOT EDIT MANUALLY - use std::borrow::Cow; -use serde::{Deserialize, Serialize}; - - -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct EndpointTool { - pub name: Cow<'static, str>, - pub description: Cow<'static, str>, - pub instructions: Cow<'static, str>, - pub path: Cow<'static, str>, - pub method: Cow<'static, str>, - pub path_params_schema: Option, - pub query_params_schema: Option, - pub body_schema: Option, -} +use windmill_mcp::server::EndpointTool; pub fn all_tools() -> Vec { vec![ diff --git a/backend/windmill-api/src/mcp/core.rs b/backend/windmill-api/src/mcp/core.rs new file mode 100644 index 0000000000..6b5917398e --- /dev/null +++ b/backend/windmill-api/src/mcp/core.rs @@ -0,0 +1,448 @@ +//! Windmill MCP Backend implementation +//! +//! This module provides the concrete implementation of the McpBackend trait +//! for the Windmill platform. + +use async_trait::async_trait; +use serde_json::Value; +use std::collections::HashMap; +use windmill_common::{db::UserDB, utils::StripPath, DB}; +use windmill_mcp::common::transform::apply_key_transformation; +use windmill_mcp::common::types::{ + FlowInfo, HubScriptInfo, ResourceInfo, ResourceType, SchemaType, ScriptInfo, +}; +use windmill_mcp::server::{BackendResult, EndpointTool, ErrorData, McpAuth, McpBackend}; + +use crate::db::ApiAuthed; +use crate::jobs::{ + run_wait_result_flow_by_path_internal, run_wait_result_script_by_path_internal, RunJobQuery, +}; + +use super::auto_generated_endpoints::all_tools; +use super::utils::{ + build_query_string, build_request_body, create_http_request, get_hub_script_schema, + get_item_schema, get_items, get_resources, get_resources_types, get_scripts_from_hub, + parse_response_body, prepare_push_args, substitute_path_params, +}; + +use std::sync::Arc; +use std::time::Duration; + +use tokio_util::sync::CancellationToken; +use windmill_mcp::server::{ + LocalSessionManager, Runner, StreamableHttpServerConfig, StreamableHttpService, +}; +use windmill_mcp::WorkspaceId; + +use axum::{ + extract::Path, http::Request, middleware::Next, response::Response, routing::get, Json, Router, +}; +use windmill_common::error::JsonResult; + +/// Implement McpAuth for ApiAuthed +impl McpAuth for ApiAuthed { + fn username(&self) -> &str { + &self.username + } + + fn email(&self) -> &str { + &self.email + } + + fn is_admin(&self) -> bool { + self.is_admin + } + + fn is_operator(&self) -> bool { + self.is_operator + } + + fn groups(&self) -> &[String] { + &self.groups + } + + fn folders(&self) -> &[(String, bool, bool)] { + &self.folders + } + + fn scopes(&self) -> Option<&[String]> { + self.scopes.as_deref() + } +} + +/// Windmill's MCP backend implementation +#[derive(Clone)] +pub struct WindmillBackend { + pub db: DB, + pub user_db: UserDB, +} + +impl WindmillBackend { + pub fn new(db: DB, user_db: UserDB) -> Self { + Self { db, user_db } + } +} + +#[async_trait] +impl McpBackend for WindmillBackend { + type Auth = ApiAuthed; + + async fn list_scripts( + &self, + auth: &ApiAuthed, + workspace_id: &str, + favorites_only: bool, + ) -> BackendResult> { + let scope_type = if favorites_only { "favorites" } else { "all" }; + get_items::(&self.user_db, auth, workspace_id, scope_type, "script") + .await + .map_err(|e| ErrorData::internal_error(e.message, None)) + } + + async fn list_flows( + &self, + auth: &ApiAuthed, + workspace_id: &str, + favorites_only: bool, + ) -> BackendResult> { + let scope_type = if favorites_only { "favorites" } else { "all" }; + get_items::(&self.user_db, auth, workspace_id, scope_type, "flow") + .await + .map_err(|e| ErrorData::internal_error(e.message, None)) + } + + async fn list_resource_types( + &self, + auth: &ApiAuthed, + workspace_id: &str, + ) -> BackendResult> { + get_resources_types(&self.user_db, auth, workspace_id) + .await + .map_err(|e| ErrorData::internal_error(e.message, None)) + } + + async fn list_resources( + &self, + auth: &ApiAuthed, + workspace_id: &str, + resource_type: &str, + ) -> BackendResult> { + get_resources(&self.user_db, auth, workspace_id, resource_type) + .await + .map_err(|e| ErrorData::internal_error(e.message, None)) + } + + async fn list_hub_scripts( + &self, + app_filter: Option<&str>, + ) -> BackendResult> { + get_scripts_from_hub(&self.db, app_filter) + .await + .map_err(|e| ErrorData::internal_error(e.message, None)) + } + + async fn get_item_schema( + &self, + auth: &ApiAuthed, + workspace_id: &str, + path: &str, + item_type: &str, + ) -> BackendResult> { + let schema = get_item_schema(path, &self.user_db, auth, workspace_id, item_type) + .await + .map_err(|e| ErrorData::internal_error(e.message, None))?; + + if let Some(ref s) = schema { + match serde_json::from_str::(s.0.get()) { + Ok(val) => Ok(Some(val)), + Err(e) => { + tracing::warn!("Failed to parse schema: {}", e); + Ok(None) + } + } + } else { + Ok(None) + } + } + + async fn get_hub_script_schema(&self, path: &str) -> BackendResult> { + let schema = get_hub_script_schema(path, &self.db) + .await + .map_err(|e| ErrorData::internal_error(e.message, None))?; + + if let Some(ref s) = schema { + match serde_json::from_str::(s.0.get()) { + Ok(val) => Ok(Some(val)), + Err(e) => { + tracing::warn!("Failed to parse hub schema: {}", e); + Ok(None) + } + } + } else { + Ok(None) + } + } + + fn transform_schema_for_resources( + &self, + schema: &SchemaType, + resources_cache: &HashMap>, + resources_types: &[ResourceType], + ) -> SchemaType { + let mut schema_obj = schema.clone(); + + // Replace invalid char in property key with underscore + let replacements: Vec<(String, String, Value)> = schema_obj + .properties + .iter() + .filter_map(|(key, value)| { + if key.chars().any(|c| !c.is_alphanumeric() && c != '_') { + let new_key = apply_key_transformation(key); + Some((key.clone(), new_key, value.clone())) + } else { + None + } + }) + .collect(); + + for (old_key, new_key, value) in replacements { + schema_obj.properties.remove(&old_key); + schema_obj.properties.insert(new_key, value); + } + + for (_key, prop_value) in schema_obj.properties.iter_mut() { + if let Value::Object(prop_map) = prop_value { + if let Some(format_value) = prop_map.get("format") { + if let Value::String(format_str) = format_value { + if format_str.starts_with("resource-") { + let resource_type_key = + format_str.split("-").last().unwrap_or_default().to_string(); + let resource_type = resources_types + .iter() + .find(|rt| rt.name == resource_type_key); + let resource_type_obj = resource_type.cloned(); + + if let Some(resource_cache) = resources_cache.get(&resource_type_key) { + let resources_count = resource_cache.len(); + let description = match resource_type_obj { + Some(resource_type_obj) => format!( + "This is a resource named `{}` with the following description: `{}`.\\nThe path of the resource should be used to specify the resource.\\n{}", + resource_type_obj.name, + resource_type_obj.description.as_deref().unwrap_or("No description"), + if resources_count == 0 { + "This resource does not have any available instances, you should create one from your windmill workspace." + } else if resources_count > 1 { + "This resource has multiple available instances, you should precisely select the one you want to use." + } else { + "There is 1 resource available." + } + ), + None => "An object parameter.".to_string(), + }; + prop_map.insert( + "type".to_string(), + Value::String("string".to_string()), + ); + prop_map + .insert("description".to_string(), Value::String(description)); + if resources_count > 0 { + let resources_description = resource_cache + .iter() + .map(|resource| { + format!( + "{}: $res:{}", + resource + .description + .as_deref() + .unwrap_or("No title"), + resource.path + ) + }) + .collect::>() + .join("\\n"); + + prop_map.insert( + "description".to_string(), + Value::String(format!( + "{}\\nHere are the available resources, in the format title:path. Title can be empty. Path should be used to specify the resource:\\n{}", + prop_map.get("description").unwrap_or(&Value::String("No description".to_string())), + resources_description + )), + ); + } + } + } + } + } + } + } + + schema_obj + } + + async fn run_script( + &self, + auth: &ApiAuthed, + workspace_id: &str, + path: &str, + args: Value, + ) -> BackendResult { + let push_args = prepare_push_args(args); + + let result = run_wait_result_script_by_path_internal( + self.db.clone(), + RunJobQuery::default(), + StripPath(path.to_string()), + auth.clone(), + self.user_db.clone(), + workspace_id.to_string(), + push_args, + ) + .await + .map_err(|e| ErrorData::internal_error(e.to_string(), None))?; + + parse_response_body(result).await + } + + async fn run_flow( + &self, + auth: &ApiAuthed, + workspace_id: &str, + path: &str, + args: Value, + ) -> BackendResult { + let push_args = prepare_push_args(args); + + let result = run_wait_result_flow_by_path_internal( + self.db.clone(), + RunJobQuery::default(), + StripPath(path.to_string()), + auth.clone(), + self.user_db.clone(), + push_args, + workspace_id.to_string(), + ) + .await + .map_err(|e| ErrorData::internal_error(e.to_string(), None))?; + + parse_response_body(result).await + } + + async fn call_endpoint( + &self, + auth: &ApiAuthed, + workspace_id: &str, + endpoint_tool: &EndpointTool, + args: Value, + ) -> BackendResult { + let args_map = match &args { + Value::Object(map) => map, + _ => { + return Err(ErrorData::invalid_params( + "Arguments must be an object", + None, + )); + } + }; + + // Build URL with path substitutions + let path_template = substitute_path_params( + &endpoint_tool.path, + workspace_id, + args_map, + &endpoint_tool.path_params_schema, + )?; + let query_string = build_query_string(args_map, &endpoint_tool.query_params_schema); + let full_url = format!( + "{}/api{}{}", + windmill_common::BASE_INTERNAL_URL.as_str(), + path_template, + query_string + ); + + // Prepare request body + let body_json = + build_request_body(&endpoint_tool.method, args_map, &endpoint_tool.body_schema); + + // Create and execute request + let response = create_http_request( + &endpoint_tool.method, + &full_url, + workspace_id, + auth, + body_json, + ) + .await?; + + let status = response.status(); + let response_text = response.text().await.map_err(|e| { + ErrorData::internal_error(format!("Failed to read response text: {}", e), None) + })?; + + if status.is_success() { + Ok(serde_json::from_str(&response_text) + .unwrap_or_else(|_| Value::String(response_text))) + } else { + Err(ErrorData::internal_error( + format!( + "HTTP {} {}: {}", + status.as_u16(), + status.canonical_reason().unwrap_or(""), + response_text + ), + None, + )) + } + } + + fn all_endpoint_tools(&self) -> Vec { + all_tools() + } +} + +/// Extract workspace ID from path and store it in request extensions +pub async fn extract_and_store_workspace_id( + Path(params): Path, + mut request: Request, + next: Next, +) -> Response { + let workspace_id = params; + request.extensions_mut().insert(WorkspaceId(workspace_id)); + next.run(request).await +} + +/// Setup the MCP server with HTTP transport +pub async fn setup_mcp_server( + db: DB, + user_db: UserDB, +) -> anyhow::Result<(Router, CancellationToken)> { + let cancellation_token = CancellationToken::new(); + let session_manager = Arc::new(LocalSessionManager::default()); + + let backend = WindmillBackend::new(db, user_db); + let runner = Runner::new(backend); + + let service_config = StreamableHttpServerConfig { + sse_keep_alive: Some(Duration::from_secs(15)), + stateful_mode: false, + cancellation_token: cancellation_token.clone(), + sse_retry: Some(Duration::from_secs(15)), + }; + + let service = + StreamableHttpService::new(move || Ok(runner.clone()), session_manager, service_config); + + let router = Router::new().nest_service("/", service); + Ok((router, cancellation_token)) +} + +/// HTTP handler to list MCP tools as JSON +async fn list_mcp_tools_handler() -> JsonResult> { + let endpoint_tools = all_tools(); + Ok(Json(endpoint_tools)) +} + +/// Creates a router service for listing MCP tools +pub fn list_tools_service() -> Router { + Router::new().route("/", get(list_mcp_tools_handler)) +} diff --git a/backend/windmill-api/src/mcp/mod.rs b/backend/windmill-api/src/mcp/mod.rs index e1cd855547..b7853fd7de 100644 --- a/backend/windmill-api/src/mcp/mod.rs +++ b/backend/windmill-api/src/mcp/mod.rs @@ -3,9 +3,9 @@ //! This module provides the MCP server implementation that exposes Windmill scripts, //! flows, and API endpoints as MCP tools for AI assistants to interact with. -pub mod server; -pub mod tools; -pub mod utils; +mod auto_generated_endpoints; +mod core; +mod utils; -// Re-export main components -pub use server::{extract_and_store_workspace_id, list_tools_service, setup_mcp_server}; +// Re-export only what's needed externally +pub use core::{extract_and_store_workspace_id, list_tools_service, setup_mcp_server}; diff --git a/backend/windmill-api/src/mcp/server.rs b/backend/windmill-api/src/mcp/server.rs deleted file mode 100644 index ba38de4b1c..0000000000 --- a/backend/windmill-api/src/mcp/server.rs +++ /dev/null @@ -1,561 +0,0 @@ -//! MCP Server implementation -//! -//! Contains the core MCP server handler that implements the Model Context Protocol -//! specification. This is a thin orchestration layer that delegates to the appropriate -//! modules for tool management, database operations, and schema transformation. - -use std::collections::HashMap; -use std::sync::Arc; -use std::{borrow::Cow, time::Duration}; - -use axum::body::to_bytes; -use serde_json::Value; -use tokio::try_join; -use tokio_util::sync::CancellationToken; -use windmill_common::db::UserDB; -use windmill_common::worker::to_raw_value; -use windmill_common::{utils::StripPath, DB}; -use windmill_mcp::server::{ - Annotated, CallToolRequestParam, CallToolResult, Content, ErrorData, Implementation, - InitializeRequestParam, InitializeResult, ListPromptsResult, ListResourceTemplatesResult, - ListResourcesResult, ListToolsResult, LocalSessionManager, PaginatedRequestParam, - ProtocolVersion, RawContent, RawTextContent, RequestContext, RoleServer, ServerCapabilities, - ServerHandler, ServerInfo, StreamableHttpServerConfig, StreamableHttpService, Tool, - ToolAnnotations, -}; - -use crate::db::ApiAuthed; -use crate::jobs::{ - run_wait_result_flow_by_path_internal, run_wait_result_script_by_path_internal, RunJobQuery, -}; - -use super::tools::endpoint_tools::{ - all_endpoint_tools, call_endpoint_tool, endpoint_tools_to_mcp_tools, EndpointTool, -}; -use super::utils::{ - database::{ - check_scopes, get_hub_script_schema, get_item_schema, get_items, get_resources_types, - get_scripts_from_hub, - }, - models::{ - FlowInfo, ResourceInfo, ResourceType, SchemaType, ScriptInfo, ToolableItem, WorkspaceId, - }, - schema::transform_schema_for_resources, - scope_matcher::{is_resource_allowed, parse_mcp_scopes}, - transform::{reverse_transform, reverse_transform_key}, -}; - -use axum::{ - extract::Path, http::Request, middleware::Next, response::Response, routing::get, Json, Router, -}; -use windmill_common::error::JsonResult; - -/// MCP Server Runner - implements the core MCP protocol handlers -#[derive(Clone)] -pub struct Runner {} - -impl Runner { - pub fn new() -> Self { - Self {} - } - - /// Creates a Tool from a ToolableItem - async fn create_tool_from_item( - item: &T, - user_db: &UserDB, - authed: &ApiAuthed, - workspace_id: &str, - resources_cache: &mut HashMap>, - resources_types: &Vec, - ) -> Result { - let is_hub = item.is_hub(); - let path = item.get_path_or_id(); - let item_type = item.item_type(); - let description = format!( - "This is a {} named `{}` with the following description: `{}`.{}", - item_type, - item.get_summary(), - item.get_description(), - if is_hub { - format!( - " It is a tool used for the following app: {}", - item.get_integration_type() - .unwrap_or("No integration type".to_string()) - ) - } else { - "".to_string() - } - ); - let schema_obj = transform_schema_for_resources( - &item.get_schema(), - user_db, - authed, - &workspace_id, - resources_cache, - &resources_types, - ) - .await?; - let input_schema_map = match serde_json::to_value(schema_obj) { - Ok(Value::Object(map)) => map, - Ok(_) => { - tracing::warn!("Schema object for tool '{}' did not serialize to a JSON object, using empty schema.", path); - serde_json::Map::new() - } - Err(e) => { - tracing::error!( - "Failed to serialize schema object for tool '{}': {}. Using empty schema.", - path, - e - ); - serde_json::Map::new() - } - }; - - Ok(Tool { - name: Cow::Owned(path), - description: Some(Cow::Owned(description)), - input_schema: Arc::new(input_schema_map), - title: Some(item.get_summary().to_string()), - output_schema: None, - icons: None, - annotations: Some(ToolAnnotations { - title: Some(item.get_summary().to_string()), - read_only_hint: Some(false), // Can modify environment - destructive_hint: Some(true), // Can potentially be destructive - idempotent_hint: Some(false), // Are not guaranteed to be idempotent - open_world_hint: Some(true), // Can interact with external services - }), - meta: None, - }) - } -} - -impl ServerHandler for Runner { - /// Handles the `CallTool` request from the MCP client - async fn call_tool( - &self, - request: CallToolRequestParam, - context: RequestContext, - ) -> Result { - let http_parts = context - .extensions - .get::() - .ok_or_else(|| { - tracing::error!("http::request::Parts not found"); - ErrorData::internal_error("http::request::Parts not found", None) - })?; - - let authed = http_parts.extensions.get::().ok_or_else(|| { - tracing::error!("ApiAuthed Axum extension not found"); - ErrorData::internal_error("ApiAuthed Axum extension not found", None) - })?; - - check_scopes(authed)?; - - // Parse MCP scopes for authorization - let scopes = authed.scopes.as_ref().map(|s| s.as_slice()).unwrap_or(&[]); - let scope_config = parse_mcp_scopes(scopes)?; - - if request.name.ends_with("_TRUNC") { - return Ok(CallToolResult::error( - vec![ - Annotated::new( - RawContent::Text(RawTextContent { - text: - "Tool path is too long. Consider shortening it to make it compatible with MCP." - .to_string(), - meta: None, - }), - None - ), - ] - )); - } - - let db = http_parts.extensions.get::().ok_or_else(|| { - tracing::error!("DB Axum extension not found"); - ErrorData::internal_error("DB Axum extension not found", None) - })?; - - let user_db = http_parts.extensions.get::().ok_or_else(|| { - tracing::error!("UserDB Axum extension not found"); - ErrorData::internal_error("UserDB Axum extension not found", None) - })?; - - let args = request.arguments.map(Value::Object).ok_or_else(|| { - ErrorData::invalid_params( - "Missing arguments for tool", - Some(request.name.clone().into()), - ) - })?; - - let workspace_id = http_parts - .extensions - .get::() - .ok_or_else(|| { - tracing::error!("WorkspaceId not found"); - ErrorData::internal_error("WorkspaceId not found", None) - }) - .map(|w_id| w_id.0.clone())?; - - // Check if this is a generated endpoint tool - let endpoint_tools = all_endpoint_tools(); - for endpoint_tool in endpoint_tools { - if endpoint_tool.name.as_ref() == request.name { - // Validate endpoint scope - if scope_config.granular - && !is_resource_allowed(&endpoint_tool.name, &scope_config.endpoints) - { - return Err(ErrorData::internal_error( - format!( - "Access denied: endpoint '{}' not in token scope", - endpoint_tool.name - ), - None, - )); - } - - // This is an endpoint tool, forward to the actual HTTP endpoint - let result = - call_endpoint_tool(&endpoint_tool, args.clone(), &workspace_id, &authed) - .await?; - return Ok(CallToolResult::success(vec![Content::text( - serde_json::to_string_pretty(&result).unwrap_or_else(|_| "{}".to_string()), - )])); - } - } - - // Continue with script/flow logic - let (tool_type, path, is_hub) = reverse_transform(&request.name).map_err(|e| { - ErrorData::internal_error(format!("Failed to reverse transform path: {}", e), None) - })?; - - // Validate script/flow scope - if !is_hub && scope_config.granular { - if tool_type == "script" && !is_resource_allowed(&path, &scope_config.scripts) { - return Err(ErrorData::internal_error( - format!("Access denied: script '{}' not in token scope", path), - None, - )); - } else if tool_type == "flow" && !is_resource_allowed(&path, &scope_config.flows) { - return Err(ErrorData::internal_error( - format!("Access denied: flow '{}' not in token scope", path), - None, - )); - } - } - - let item_schema = if is_hub { - get_hub_script_schema(&format!("hub/{}", path), db).await? - } else { - get_item_schema(&path, user_db, authed, &workspace_id, &tool_type).await? - }; - - let schema_obj = if let Some(ref s) = item_schema { - match serde_json::from_str::(s.0.get()) { - Ok(val) => Some(val), - Err(e) => { - tracing::warn!("Failed to parse schema: {}", e); - None - } - } - } else { - None - }; - - let push_args = if let Value::Object(map) = args.clone() { - let mut args_hash = HashMap::new(); - for (k, v) in map { - // need to transform back the key without invalid characters to the original key - let original_key = reverse_transform_key(&k, &schema_obj); - args_hash.insert(original_key, to_raw_value(&v)); - } - windmill_queue::PushArgsOwned { extra: None, args: args_hash } - } else { - windmill_queue::PushArgsOwned::default() - }; - let script_or_flow_path = if is_hub { - StripPath(format!("hub/{}", path)) - } else { - StripPath(path) - }; - let run_query = RunJobQuery::default(); - - let result = if tool_type == "script" { - run_wait_result_script_by_path_internal( - db.clone(), - run_query, - script_or_flow_path, - authed.clone(), - user_db.clone(), - workspace_id.clone(), - push_args, - ) - .await - } else { - run_wait_result_flow_by_path_internal( - db.clone(), - run_query, - script_or_flow_path, - authed.clone(), - user_db.clone(), - push_args, - workspace_id.clone(), - ) - .await - }; - - match result { - Ok(response) => { - let body_bytes = to_bytes(response.into_body(), usize::MAX) - .await - .map_err(|e| { - ErrorData::internal_error( - format!("Failed to read response body: {}", e), - None, - ) - })?; - let body_str = String::from_utf8(body_bytes.to_vec()).map_err(|e| { - ErrorData::internal_error( - format!("Failed to decode response body: {}", e), - None, - ) - })?; - Ok(CallToolResult::success(vec![Content::text(body_str)])) - } - Err(e) => Err(ErrorData::internal_error( - format!("Failed to run script: {}", e), - None, - )), - } - } - - /// Fetches available tools (scripts, flows, hub scripts) based on the user's scope - async fn list_tools( - &self, - _request: Option, - mut _context: RequestContext, - ) -> Result { - let http_parts = _context - .extensions - .get::() - .ok_or_else(|| { - tracing::error!("http::request::Parts not found"); - ErrorData::internal_error("http::request::Parts not found", None) - })?; - - let authed = http_parts.extensions.get::().ok_or_else(|| { - tracing::error!("ApiAuthed Axum extension not found"); - ErrorData::internal_error("ApiAuthed Axum extension not found", None) - })?; - - check_scopes(authed)?; - - let db = http_parts.extensions.get::().ok_or_else(|| { - tracing::error!("DB Axum extension not found"); - ErrorData::internal_error("DB Axum extension not found", None) - })?; - - let user_db = http_parts.extensions.get::().ok_or_else(|| { - tracing::error!("UserDB Axum extension not found"); - ErrorData::internal_error("UserDB Axum extension not found", None) - })?; - - let workspace_id = http_parts - .extensions - .get::() - .ok_or_else(|| { - tracing::error!("WorkspaceId not found"); - ErrorData::internal_error("WorkspaceId not found", None) - }) - .map(|w_id| w_id.0.clone())?; - - // Parse MCP scopes to determine what to expose - let scopes = authed.scopes.as_ref().map(|s| s.as_slice()).unwrap_or(&[]); - let scope_config = parse_mcp_scopes(scopes)?; - - let scope_type = if scope_config.favorites { - "favorites" - } else { - // Fetch all items if either all or granular scope set (we filter later for granular scopes) - "all" - }; - - let scripts_fn = - get_items::(user_db, authed, &workspace_id, scope_type, "script"); - let flows_fn = get_items::(user_db, authed, &workspace_id, scope_type, "flow"); - let resources_types_fn = get_resources_types(user_db, authed, &workspace_id); - let hub_scripts_fn = get_scripts_from_hub(db, scope_config.hub_apps.as_deref()); - let (scripts, flows, resources_types, hub_scripts) = if scope_config.hub_apps.is_some() { - let (scripts, flows, resources_types, hub_scripts) = - try_join!(scripts_fn, flows_fn, resources_types_fn, hub_scripts_fn)?; - (scripts, flows, resources_types, hub_scripts) - } else { - let (scripts, flows, resources_types) = - try_join!(scripts_fn, flows_fn, resources_types_fn)?; - (scripts, flows, resources_types, vec![]) - }; - - let mut resources_cache: HashMap> = HashMap::new(); - let mut tools: Vec = Vec::new(); - - // Filter and add scripts based on scope - for script in scripts { - // For granular scopes, filter by path - if scope_config.granular && !is_resource_allowed(&script.path, &scope_config.scripts) { - continue; - } - - tools.push( - Runner::create_tool_from_item( - &script, - user_db, - authed, - &workspace_id, - &mut resources_cache, - &resources_types, - ) - .await?, - ); - } - - // Filter and add flows based on scope - for flow in flows { - // For granular scopes, filter by path - if scope_config.granular && !is_resource_allowed(&flow.path, &scope_config.flows) { - continue; - } - - tools.push( - Runner::create_tool_from_item( - &flow, - user_db, - authed, - &workspace_id, - &mut resources_cache, - &resources_types, - ) - .await?, - ); - } - - for hub_script in hub_scripts { - tools.push( - Runner::create_tool_from_item( - &hub_script, - user_db, - authed, - &workspace_id, - &mut resources_cache, - &resources_types, - ) - .await?, - ); - } - - // Add endpoint tools from the generated MCP tools, filtered by scope - let endpoint_tools = all_endpoint_tools(); - for endpoint_tool in endpoint_tools { - // For granular scopes, filter by endpoint name - if scope_config.granular - && !is_resource_allowed(&endpoint_tool.name, &scope_config.endpoints) - { - continue; - } - - tools.push( - endpoint_tools_to_mcp_tools(vec![endpoint_tool]) - .into_iter() - .next() - .unwrap(), - ); - } - - Ok(ListToolsResult { tools, next_cursor: None, meta: None }) - } - - fn get_info(&self) -> ServerInfo { - ServerInfo { - protocol_version: ProtocolVersion::default(), - capabilities: ServerCapabilities::builder() - .enable_tools() - .build(), - server_info: Implementation::from_build_env(), - instructions: Some("This server provides a list of scripts and flows the user can run on Windmill. Each flow and script is a tool callable with their respective arguments.".to_string()), - } - } - - async fn initialize( - &self, - _request: InitializeRequestParam, - _context: RequestContext, - ) -> Result { - Ok(self.get_info()) - } - - async fn list_resources( - &self, - _request: Option, - _context: RequestContext, - ) -> Result { - Ok(ListResourcesResult { resources: vec![], next_cursor: None, meta: None }) - } - - async fn list_prompts( - &self, - _request: Option, - _context: RequestContext, - ) -> Result { - Ok(ListPromptsResult::default()) - } - - async fn list_resource_templates( - &self, - _request: Option, - _context: RequestContext, - ) -> Result { - Ok(ListResourceTemplatesResult::default()) - } -} - -/// Extract workspace ID from path and store it in request extensions -pub async fn extract_and_store_workspace_id( - Path(params): Path, - mut request: Request, - next: Next, -) -> Response { - let workspace_id = params; - request.extensions_mut().insert(WorkspaceId(workspace_id)); - next.run(request).await -} - -/// Setup the MCP server with HTTP transport -pub async fn setup_mcp_server() -> anyhow::Result<(Router, CancellationToken)> { - let cancellation_token = CancellationToken::new(); - let session_manager = Arc::new(LocalSessionManager::default()); - let service_config = StreamableHttpServerConfig { - sse_keep_alive: Some(Duration::from_secs(15)), - stateful_mode: false, - cancellation_token: cancellation_token.clone(), - sse_retry: Some(Duration::from_secs(15)), - }; - let service = StreamableHttpService::new( - || Ok(Runner::new()), - session_manager.clone(), - service_config, - ); - - let router = axum::Router::new().nest_service("/", service); - Ok((router, cancellation_token)) -} - -/// HTTP handler to list MCP tools as JSON -async fn list_mcp_tools_handler() -> JsonResult> { - let endpoint_tools = all_endpoint_tools(); - Ok(Json(endpoint_tools)) -} - -/// Creates a router service for listing MCP tools -pub fn list_tools_service() -> Router { - Router::new().route("/", get(list_mcp_tools_handler)) -} diff --git a/backend/windmill-api/src/mcp/tools/endpoint_tools.rs b/backend/windmill-api/src/mcp/tools/endpoint_tools.rs deleted file mode 100644 index 372310b463..0000000000 --- a/backend/windmill-api/src/mcp/tools/endpoint_tools.rs +++ /dev/null @@ -1,307 +0,0 @@ -//! Endpoint tools for MCP server -//! -//! Contains the auto-generated endpoint tools and utilities for converting -//! them to MCP tools and handling HTTP calls to Windmill API endpoints. - -use crate::db::ApiAuthed; -use std::sync::Arc; -use windmill_common::db::Authed; -use windmill_common::{auth::create_jwt_token, BASE_INTERNAL_URL}; -use windmill_mcp::server::{ErrorData, Tool, ToolAnnotations}; - -// Import the auto-generated tools -use super::auto_generated_endpoints; -pub use auto_generated_endpoints::{all_tools, EndpointTool}; - -/// Get all available endpoint tools -pub fn all_endpoint_tools() -> Vec { - all_tools() -} - -/// Convert endpoint tools to MCP tools -pub fn endpoint_tools_to_mcp_tools(endpoint_tools: Vec) -> Vec { - endpoint_tools - .into_iter() - .map(|tool| endpoint_tool_to_mcp_tool(&tool)) - .collect() -} - -/// Convert a single endpoint tool to MCP tool -pub fn endpoint_tool_to_mcp_tool(tool: &EndpointTool) -> Tool { - let mut combined_properties = serde_json::Map::new(); - let mut combined_required = Vec::new(); - - // Combine all parameter schemas - let schemas = [ - &tool.path_params_schema, - &tool.query_params_schema, - &tool.body_schema, - ]; - - for schema in schemas.iter().filter_map(|s| s.as_ref()) { - merge_schema_into(&mut combined_properties, &mut combined_required, schema); - } - - let combined_schema = serde_json::json!({ - "type": "object", - "properties": combined_properties, - "required": combined_required - }); - - let description = format!("{}. {}", tool.description, tool.instructions); - - // Create annotations based on HTTP method and endpoint characteristics - let annotations = create_endpoint_annotations(tool); - - Tool { - name: tool.name.clone(), - description: Some(description.into()), - input_schema: Arc::new(combined_schema.as_object().unwrap().clone()), - title: Some(tool.name.to_string()), - output_schema: None, - icons: None, - annotations: Some(annotations), - meta: None, - } -} - -/// Create appropriate annotations for endpoint tools based on HTTP method -fn create_endpoint_annotations(tool: &EndpointTool) -> ToolAnnotations { - let method = tool.method.as_ref(); - - // Determine characteristics based on HTTP method - let (read_only, destructive, idempotent, open_world) = match method { - "GET" => (true, false, true, true), // Read-only, safe, idempotent - "POST" => (false, true, false, true), // Can modify, potentially destructive, not idempotent - "PUT" => (false, false, true, true), // Can modify, typically idempotent updates - "DELETE" => (false, true, true, true), // Destructive but idempotent - "PATCH" => (false, false, false, true), // Partial updates, not guaranteed idempotent - _ => (false, true, false, true), // Default: assume can modify and be destructive - }; - - ToolAnnotations { - title: Some(format!("{} {}", method, tool.path)), - read_only_hint: Some(read_only), - destructive_hint: Some(destructive), - idempotent_hint: Some(idempotent), - open_world_hint: Some(open_world), - } -} - -/// Merge schema into combined properties and required fields -fn merge_schema_into( - combined_properties: &mut serde_json::Map, - combined_required: &mut Vec, - schema: &serde_json::Value, -) { - if let Some(props) = schema.get("properties").and_then(|p| p.as_object()) { - for (key, value) in props { - combined_properties.insert(key.clone(), value.clone()); - } - } - - if let Some(required) = schema.get("required").and_then(|r| r.as_array()) { - for req in required.iter().filter_map(|r| r.as_str()) { - combined_required.push(req.to_string()); - } - } -} - -/// Call an endpoint tool by making HTTP request to Windmill API -pub async fn call_endpoint_tool( - tool: &EndpointTool, - args: serde_json::Value, - workspace_id: &str, - api_authed: &ApiAuthed, -) -> Result { - let args_map = match &args { - serde_json::Value::Object(map) => map, - _ => { - return Err(ErrorData::invalid_params( - "Arguments must be an object", - Some(tool.name.clone().into()), - )) - } - }; - - // Build URL with path substitutions - let path_template = - substitute_path_params(&tool.path, workspace_id, args_map, &tool.path_params_schema)?; - let query_string = build_query_string(args_map, &tool.query_params_schema); - let full_url = format!( - "{}/api{}{}", - BASE_INTERNAL_URL.as_str(), - path_template, - query_string - ); - - // Prepare request body - let body_json = build_request_body(&tool.method, args_map, &tool.body_schema); - - // Create and execute request - let response = - create_http_request(&tool.method, &full_url, workspace_id, api_authed, body_json).await?; - - let status = response.status(); - let response_text = response.text().await.map_err(|e| { - ErrorData::internal_error(format!("Failed to read response text: {}", e), None) - })?; - - if status.is_success() { - Ok(serde_json::from_str(&response_text) - .unwrap_or_else(|_| serde_json::Value::String(response_text))) - } else { - Err(ErrorData::internal_error( - format!( - "HTTP {} {}: {}", - status.as_u16(), - status.canonical_reason().unwrap_or(""), - response_text - ), - None, - )) - } -} - -/// Substitute path parameters in the URL template -fn substitute_path_params( - path: &str, - workspace_id: &str, - args_map: &serde_json::Map, - path_schema: &Option, -) -> Result { - let mut path_template = path.replace("{workspace}", workspace_id); - - if let Some(schema) = path_schema { - if let Some(props) = schema.get("properties").and_then(|p| p.as_object()) { - for (param_name, _) in props { - let placeholder = format!("{{{}}}", param_name); - match args_map.get(param_name) { - Some(param_value) => { - if let Some(str_val) = param_value.as_str() { - path_template = path_template.replace(&placeholder, str_val); - } - } - None => { - tracing::warn!("Missing required path parameter: {}", param_name); - return Err(ErrorData::invalid_params( - format!("Missing required path parameter: {}", param_name), - None, - )); - } - } - } - } - } - - Ok(path_template) -} - -/// Build query string from arguments -fn build_query_string( - args_map: &serde_json::Map, - query_schema: &Option, -) -> String { - let Some(schema) = query_schema else { - return String::new(); - }; - let Some(props) = schema.get("properties").and_then(|p| p.as_object()) else { - return String::new(); - }; - - let query_params: Vec = props - .keys() - .filter_map(|param_name| { - args_map - .get(param_name) - .filter(|v| !v.is_null()) - .map(|value| { - let value_str = value.to_string(); - let str_val = value_str.trim_matches('"'); - format!( - "{}={}", - urlencoding::encode(param_name), - urlencoding::encode(str_val) - ) - }) - }) - .collect(); - - if query_params.is_empty() { - String::new() - } else { - format!("?{}", query_params.join("&")) - } -} - -/// Build request body from arguments -fn build_request_body( - method: &str, - args_map: &serde_json::Map, - body_schema: &Option, -) -> Option { - if method == "GET" { - return None; - } - - let schema = body_schema.as_ref()?; - let props = schema.get("properties")?.as_object()?; - - let body_map: serde_json::Map = props - .keys() - .filter_map(|param_name| { - args_map - .get(param_name) - .map(|value| (param_name.clone(), value.clone())) - }) - .collect(); - - if body_map.is_empty() { - None - } else { - Some(serde_json::Value::Object(body_map)) - } -} - -/// Create HTTP request with authentication -async fn create_http_request( - method: &str, - url: &str, - workspace_id: &str, - api_authed: &ApiAuthed, - body_json: Option, -) -> Result { - let client = &crate::HTTP_CLIENT; - let mut request_builder = match method { - "GET" => client.get(url), - "POST" => client.post(url), - "PUT" => client.put(url), - "DELETE" => client.delete(url), - "PATCH" => client.patch(url), - _ => { - return Err(ErrorData::invalid_params( - format!("Unsupported HTTP method: {}", method), - None, - )) - } - }; - - // Add authorization header - let authed = Authed::from(api_authed.clone()); - let token = create_jwt_token(authed, workspace_id, 3600, None, None, None, None) - .await - .map_err(|e| ErrorData::internal_error(e.to_string(), None))?; - request_builder = request_builder.header("Authorization", format!("Bearer {}", token)); - - // Add body if present - if let Some(body) = body_json { - request_builder = request_builder - .header("Content-Type", "application/json") - .json(&body); - } - - request_builder - .send() - .await - .map_err(|e| ErrorData::internal_error(format!("Failed to execute request: {}", e), None)) -} diff --git a/backend/windmill-api/src/mcp/tools/flow_tools.rs b/backend/windmill-api/src/mcp/tools/flow_tools.rs deleted file mode 100644 index 03b47472bf..0000000000 --- a/backend/windmill-api/src/mcp/tools/flow_tools.rs +++ /dev/null @@ -1,40 +0,0 @@ -//! Flow tools for MCP server -//! -//! Contains functionality for converting Windmill flows into MCP tools. - -use super::super::utils::{ - models::{FlowInfo, ToolableItem, SchemaType}, - schema::convert_schema_to_schema_type, - transform::transform_path, -}; - -/// Implementation of ToolableItem for FlowInfo -impl ToolableItem for FlowInfo { - fn get_path_or_id(&self) -> String { - transform_path(&self.path, "flow") - } - - fn get_summary(&self) -> &str { - self.summary.as_deref().unwrap_or("No summary") - } - - fn get_description(&self) -> &str { - self.description.as_deref().unwrap_or("No description") - } - - fn get_schema(&self) -> SchemaType { - convert_schema_to_schema_type(self.schema.clone()) - } - - fn is_hub(&self) -> bool { - false - } - - fn item_type(&self) -> &'static str { - "flow" - } - - fn get_integration_type(&self) -> Option { - None - } -} \ No newline at end of file diff --git a/backend/windmill-api/src/mcp/tools/hub_tools.rs b/backend/windmill-api/src/mcp/tools/hub_tools.rs deleted file mode 100644 index b81b973b30..0000000000 --- a/backend/windmill-api/src/mcp/tools/hub_tools.rs +++ /dev/null @@ -1,43 +0,0 @@ -//! Hub tools for MCP server -//! -//! Contains functionality for integrating Windmill Hub scripts as MCP tools. - -use super::super::utils::{ - models::{HubScriptInfo, ToolableItem, SchemaType}, -}; - -/// Implementation of ToolableItem for HubScriptInfo -impl ToolableItem for HubScriptInfo { - fn get_path_or_id(&self) -> String { - let id = self.version_id; - let summary = self.summary.as_deref().unwrap_or("No summary"); - format!("hs-{}-{}", id, summary.replace(" ", "_")) - } - - fn get_summary(&self) -> &str { - self.summary.as_deref().unwrap_or("No summary") - } - - fn get_description(&self) -> &str { - self.description.as_deref().unwrap_or("No description") - } - - fn get_schema(&self) -> SchemaType { - match serde_json::from_value::(self.schema.clone().unwrap_or_default()) { - Ok(schema_type) => schema_type, - Err(_) => SchemaType::default(), - } - } - - fn is_hub(&self) -> bool { - true - } - - fn item_type(&self) -> &'static str { - "script" - } - - fn get_integration_type(&self) -> Option { - self.app.clone() - } -} \ No newline at end of file diff --git a/backend/windmill-api/src/mcp/tools/mod.rs b/backend/windmill-api/src/mcp/tools/mod.rs deleted file mode 100644 index 1488b22d8a..0000000000 --- a/backend/windmill-api/src/mcp/tools/mod.rs +++ /dev/null @@ -1,10 +0,0 @@ -//! Tool management for MCP server -//! -//! This module handles the conversion of Windmill scripts, flows, and endpoints -//! into MCP tools that can be used by AI assistants. - -pub mod script_tools; -pub mod flow_tools; -pub mod hub_tools; -pub mod endpoint_tools; -pub mod auto_generated_endpoints; \ No newline at end of file diff --git a/backend/windmill-api/src/mcp/tools/script_tools.rs b/backend/windmill-api/src/mcp/tools/script_tools.rs deleted file mode 100644 index 716fae3a87..0000000000 --- a/backend/windmill-api/src/mcp/tools/script_tools.rs +++ /dev/null @@ -1,40 +0,0 @@ -//! Script tools for MCP server -//! -//! Contains functionality for converting Windmill scripts into MCP tools. - -use super::super::utils::{ - models::{ScriptInfo, ToolableItem, SchemaType}, - schema::convert_schema_to_schema_type, - transform::transform_path, -}; - -/// Implementation of ToolableItem for ScriptInfo -impl ToolableItem for ScriptInfo { - fn get_path_or_id(&self) -> String { - transform_path(&self.path, "script") - } - - fn get_summary(&self) -> &str { - self.summary.as_deref().unwrap_or("No summary") - } - - fn get_description(&self) -> &str { - self.description.as_deref().unwrap_or("No description") - } - - fn get_schema(&self) -> SchemaType { - convert_schema_to_schema_type(self.schema.clone()) - } - - fn is_hub(&self) -> bool { - false - } - - fn item_type(&self) -> &'static str { - "script" - } - - fn get_integration_type(&self) -> Option { - None - } -} \ No newline at end of file diff --git a/backend/windmill-api/src/mcp/utils.rs b/backend/windmill-api/src/mcp/utils.rs new file mode 100644 index 0000000000..a2513ef5f7 --- /dev/null +++ b/backend/windmill-api/src/mcp/utils.rs @@ -0,0 +1,414 @@ +//! Utility functions for MCP server +//! +//! Contains database query functions and HTTP request helpers +//! used by the MCP server implementation. + +use std::collections::HashMap; + +use axum::body::{to_bytes, Body}; +use axum::response::Response; +use serde_json::Value; +use sql_builder::prelude::*; +use windmill_common::auth::create_jwt_token; +use windmill_common::db::{Authed, UserDB}; +use windmill_common::scripts::{get_full_hub_script_by_path, Schema}; +use windmill_common::utils::{query_elems_from_hub, StripPath}; +use windmill_common::worker::to_raw_value; +use windmill_common::{DB, HUB_BASE_URL}; +use windmill_mcp::server::{BackendResult, ErrorData}; +use windmill_mcp::{HubResponse, HubScriptInfo, ItemSchema, ResourceInfo, ResourceType}; + +use crate::db::ApiAuthed; +use crate::HTTP_CLIENT; + +// items max limit +const ITEMS_FETCH_MAX_LIMIT: usize = 100; + +// ============================================================================ +// Database utilities +// ============================================================================ + +/// Get the schema for a specific item (script or flow) +pub async fn get_item_schema( + path: &str, + user_db: &UserDB, + authed: &ApiAuthed, + workspace_id: &str, + item_type: &str, +) -> Result, ErrorData> { + let mut sqlb = SqlBuilder::select_from(&format!("{} as o", item_type)); + sqlb.fields(&["o.schema"]); + sqlb.and_where("o.path = ?".bind(&path)); + sqlb.and_where("o.workspace_id = ?".bind(&workspace_id)); + sqlb.and_where("o.archived = false"); + sqlb.and_where("o.draft_only IS NOT TRUE"); + let sql = sqlb.sql().map_err(|e| { + tracing::error!("failed to build sql: {}", e); + ErrorData::internal_error(format!("failed to build sql: {}", e), None) + })?; + let mut tx = user_db.clone().begin(authed).await.map_err(|e| { + tracing::error!("failed to begin transaction: {}", e); + ErrorData::internal_error(format!("failed to begin transaction: {}", e), None) + })?; + let item = sqlx::query_as::<_, ItemSchema>(&sql) + .fetch_one(&mut *tx) + .await + .map_err(|e| { + tracing::error!("failed to fetch item schema: {}", e); + ErrorData::internal_error(format!("failed to fetch item schema: {}", e), None) + })?; + tx.commit().await.map_err(|e| { + tracing::error!("failed to commit transaction: {}", e); + ErrorData::internal_error(format!("failed to commit transaction: {}", e), None) + })?; + Ok(item.schema) +} + +/// Get all resource types from the database +pub async fn get_resources_types( + user_db: &UserDB, + authed: &ApiAuthed, + workspace_id: &str, +) -> Result, ErrorData> { + let mut sqlb = SqlBuilder::select_from("resource_type as o"); + sqlb.fields(&["o.name", "o.description"]); + sqlb.and_where("o.workspace_id = ?".bind(&workspace_id)); + let sql = sqlb.sql().map_err(|e| { + tracing::error!("failed to build sql: {}", e); + ErrorData::internal_error(format!("failed to build sql: {}", e), None) + })?; + let mut tx = user_db.clone().begin(authed).await.map_err(|e| { + tracing::error!("failed to begin transaction: {}", e); + ErrorData::internal_error(format!("failed to begin transaction: {}", e), None) + })?; + let rows = sqlx::query_as::<_, ResourceType>(&sql) + .fetch_all(&mut *tx) + .await + .map_err(|e| { + tracing::error!("failed to fetch resource types: {}", e); + ErrorData::internal_error(format!("failed to fetch resource types: {}", e), None) + })?; + tx.commit().await.map_err(|e| { + tracing::error!("failed to commit transaction: {}", e); + ErrorData::internal_error(format!("failed to commit transaction: {}", e), None) + })?; + Ok(rows) +} + +/// Get resources by type from the database +pub async fn get_resources( + user_db: &UserDB, + authed: &ApiAuthed, + workspace_id: &str, + resource_type: &str, +) -> Result, ErrorData> { + let mut sqlb = SqlBuilder::select_from("resource as o"); + sqlb.fields(&["o.path", "o.description", "o.resource_type"]); + sqlb.and_where("o.workspace_id = ?".bind(&workspace_id)); + sqlb.and_where("o.resource_type = ?".bind(&resource_type)); + let sql = sqlb.sql().map_err(|e| { + tracing::error!("failed to build sql: {}", e); + ErrorData::internal_error(format!("failed to build sql: {}", e), None) + })?; + let mut tx = user_db.clone().begin(authed).await.map_err(|e| { + tracing::error!("failed to begin transaction: {}", e); + ErrorData::internal_error(format!("failed to begin transaction: {}", e), None) + })?; + let rows = sqlx::query_as::<_, ResourceInfo>(&sql) + .fetch_all(&mut *tx) + .await + .map_err(|e| { + tracing::error!("failed to fetch resources: {}", e); + ErrorData::internal_error(format!("failed to fetch resources: {}", e), None) + })?; + tx.commit().await.map_err(|e| { + tracing::error!("failed to commit transaction: {}", e); + ErrorData::internal_error(format!("failed to commit transaction: {}", e), None) + })?; + + Ok(rows) +} + +/// Generic function to get items (scripts or flows) from the database +pub async fn get_items sqlx::FromRow<'a, sqlx::postgres::PgRow> + Send + Unpin>( + user_db: &UserDB, + authed: &ApiAuthed, + workspace_id: &str, + scope_type: &str, + item_type: &str, +) -> Result, ErrorData> { + let mut sqlb = SqlBuilder::select_from(&format!("{} as o", item_type)); + let fields = vec!["o.path", "o.summary", "o.description", "o.schema"]; + sqlb.fields(&fields); + if scope_type == "favorites" { + sqlb.join("favorite") + .on("favorite.favorite_kind = ? AND favorite.workspace_id = o.workspace_id AND favorite.path = o.path AND favorite.usr = ?".bind(&item_type) + .bind(&authed.username)); + } + sqlb.and_where("o.workspace_id = ?".bind(&workspace_id)) + .and_where("o.archived = false") + .and_where("o.draft_only IS NOT TRUE"); + + if item_type == "script" { + sqlb.and_where("(o.no_main_func IS NOT TRUE OR o.no_main_func IS NULL)"); + } + + sqlb.order_by( + if item_type == "flow" { + "o.edited_at" + } else { + "o.created_at" + }, + false, + ) + .limit(ITEMS_FETCH_MAX_LIMIT); + let sql = sqlb.sql().map_err(|e| { + tracing::error!("failed to build sql: {}", e); + ErrorData::internal_error(format!("failed to build sql: {}", e), None) + })?; + let mut tx = user_db.clone().begin(authed).await.map_err(|e| { + tracing::error!("failed to begin transaction: {}", e); + ErrorData::internal_error(format!("failed to begin transaction: {}", e), None) + })?; + let rows = sqlx::query_as::<_, T>(&sql) + .fetch_all(&mut *tx) + .await + .map_err(|e| { + tracing::error!("failed to fetch {}: {}", item_type, e); + ErrorData::internal_error(format!("failed to fetch {}: {}", item_type, e), None) + })?; + tx.commit().await.map_err(|e| { + tracing::error!("failed to commit transaction: {}", e); + ErrorData::internal_error(format!("failed to commit transaction: {}", e), None) + })?; + Ok(rows) +} + +/// Get scripts from the Hub +pub async fn get_scripts_from_hub( + db: &DB, + scope_integrations: Option<&str>, +) -> Result, ErrorData> { + let query_params = Some(vec![ + ("limit", ITEMS_FETCH_MAX_LIMIT.to_string()), + ("with_schema", "true".to_string()), + ("apps", scope_integrations.unwrap_or("").to_string()), + ]); + let url = format!("{}/scripts/top", *HUB_BASE_URL.read().await); + let (_status_code, _headers, response) = + query_elems_from_hub(&HTTP_CLIENT, &url, query_params, &db) + .await + .map_err(|e| { + tracing::error!("Failed to get items from hub: {}", e); + ErrorData::internal_error(format!("Failed to get items from hub: {}", e), None) + })?; + + use axum::body::to_bytes; + let body_bytes = to_bytes(response, usize::MAX).await.map_err(|e| { + tracing::error!("Failed to read response body: {}", e); + ErrorData::internal_error(format!("Failed to read response body: {}", e), None) + })?; + let body_str = String::from_utf8(body_bytes.to_vec()).map_err(|e| { + tracing::error!("Failed to decode response body: {}", e); + ErrorData::internal_error(format!("Failed to decode response body: {}", e), None) + })?; + let hub_response: HubResponse = serde_json::from_str(&body_str).map_err(|e| { + tracing::error!("Failed to parse hub response: {}", e); + ErrorData::internal_error(format!("Failed to parse hub response: {}", e), None) + })?; + + Ok(hub_response.asks) +} + +/// Get the schema for a Hub script +pub async fn get_hub_script_schema(path: &str, db: &DB) -> Result, ErrorData> { + let strip_path = StripPath(path.to_string()); + let res = get_full_hub_script_by_path(strip_path, &HTTP_CLIENT, Some(db)) + .await + .map_err(|e| { + tracing::error!("Failed to get hub script: {}", e); + ErrorData::internal_error(format!("Failed to get hub script: {}", e), None) + })?; + match serde_json::from_str::(res.schema.get()) { + Ok(schema) => Ok(Some(schema)), + Err(e) => { + tracing::warn!("Failed to convert schema: {}", e); + Ok(None) + } + } +} + +// ============================================================================ +// HTTP request utilities for endpoint tools +// ============================================================================ + +/// Substitute path parameters in the URL template +pub fn substitute_path_params( + path: &str, + workspace_id: &str, + args_map: &serde_json::Map, + path_schema: &Option, +) -> BackendResult { + let mut path_template = path.replace("{workspace}", workspace_id); + + if let Some(schema) = path_schema { + if let Some(props) = schema.get("properties").and_then(|p| p.as_object()) { + for (param_name, _) in props { + let placeholder = format!("{{{}}}", param_name); + match args_map.get(param_name) { + Some(param_value) => { + if let Some(str_val) = param_value.as_str() { + path_template = path_template.replace(&placeholder, str_val); + } + } + None => { + tracing::warn!("Missing required path parameter: {}", param_name); + return Err(ErrorData::invalid_params( + format!("Missing required path parameter: {}", param_name), + None, + )); + } + } + } + } + } + + Ok(path_template) +} + +/// Build query string from arguments +pub fn build_query_string( + args_map: &serde_json::Map, + query_schema: &Option, +) -> String { + let Some(schema) = query_schema else { + return String::new(); + }; + let Some(props) = schema.get("properties").and_then(|p| p.as_object()) else { + return String::new(); + }; + + let query_params: Vec = props + .keys() + .filter_map(|param_name| { + args_map + .get(param_name) + .filter(|v| !v.is_null()) + .map(|value| { + let value_str = value.to_string(); + let str_val = value_str.trim_matches('"'); + format!( + "{}={}", + urlencoding::encode(param_name), + urlencoding::encode(str_val) + ) + }) + }) + .collect(); + + if query_params.is_empty() { + String::new() + } else { + format!("?{}", query_params.join("&")) + } +} + +/// Build request body from arguments +pub fn build_request_body( + method: &str, + args_map: &serde_json::Map, + body_schema: &Option, +) -> Option { + if method == "GET" { + return None; + } + + let schema = body_schema.as_ref()?; + let props = schema.get("properties")?.as_object()?; + + let body_map: serde_json::Map = props + .keys() + .filter_map(|param_name| { + args_map + .get(param_name) + .map(|value| (param_name.clone(), value.clone())) + }) + .collect(); + + if body_map.is_empty() { + None + } else { + Some(Value::Object(body_map)) + } +} + +/// Create HTTP request with authentication +pub async fn create_http_request( + method: &str, + url: &str, + workspace_id: &str, + api_authed: &ApiAuthed, + body_json: Option, +) -> BackendResult { + let client = &HTTP_CLIENT; + let mut request_builder = match method { + "GET" => client.get(url), + "POST" => client.post(url), + "PUT" => client.put(url), + "DELETE" => client.delete(url), + "PATCH" => client.patch(url), + _ => { + return Err(ErrorData::invalid_params( + format!("Unsupported HTTP method: {}", method), + None, + )); + } + }; + + // Add authorization header + let authed = Authed::from(api_authed.clone()); + let token = create_jwt_token(authed, workspace_id, 3600, None, None, None, None) + .await + .map_err(|e| ErrorData::internal_error(e.to_string(), None))?; + request_builder = request_builder.header("Authorization", format!("Bearer {}", token)); + + // Add body if present + if let Some(body) = body_json { + request_builder = request_builder + .header("Content-Type", "application/json") + .json(&body); + } + + request_builder + .send() + .await + .map_err(|e| ErrorData::internal_error(format!("Failed to execute request: {}", e), None)) +} + +/// Convert a JSON Value into PushArgsOwned for job execution +pub fn prepare_push_args(args: Value) -> windmill_queue::PushArgsOwned { + if let Value::Object(map) = args { + let mut args_hash = HashMap::new(); + for (k, v) in map { + args_hash.insert(k, to_raw_value(&v)); + } + windmill_queue::PushArgsOwned { extra: None, args: args_hash } + } else { + windmill_queue::PushArgsOwned::default() + } +} + +/// Parse an HTTP response body into a JSON Value +pub async fn parse_response_body(response: Response) -> BackendResult { + let body_bytes = to_bytes(response.into_body(), usize::MAX) + .await + .map_err(|e| { + ErrorData::internal_error(format!("Failed to read response body: {}", e), None) + })?; + + let body_str = String::from_utf8(body_bytes.to_vec()).map_err(|e| { + ErrorData::internal_error(format!("Failed to decode response body: {}", e), None) + })?; + + Ok(serde_json::from_str(&body_str).unwrap_or_else(|_| Value::String(body_str))) +} diff --git a/backend/windmill-api/src/mcp/utils/database.rs b/backend/windmill-api/src/mcp/utils/database.rs deleted file mode 100644 index 8453177ac0..0000000000 --- a/backend/windmill-api/src/mcp/utils/database.rs +++ /dev/null @@ -1,243 +0,0 @@ -//! Database operations for MCP server -//! -//! Contains all database query functions and database-related utilities -//! used by the MCP server implementation. - -use windmill_mcp::server::ErrorData; -use sql_builder::prelude::*; -use windmill_common::db::UserDB; -use windmill_common::scripts::{get_full_hub_script_by_path, Schema}; -use windmill_common::utils::{query_elems_from_hub, StripPath}; -use windmill_common::{DB, HUB_BASE_URL}; - -use super::models::*; -use crate::db::ApiAuthed; -use crate::HTTP_CLIENT; - -/// Check if the user has proper MCP scopes -pub fn check_scopes(authed: &ApiAuthed) -> Result<(), ErrorData> { - let scopes = authed.scopes.as_ref(); - if scopes.is_none() - || scopes - .unwrap() - .iter() - .all(|scope| !scope.starts_with("mcp:")) - { - tracing::error!("Unauthorized: missing mcp scope"); - return Err(ErrorData::internal_error( - "Unauthorized: missing mcp scope".to_string(), - None, - )); - } - Ok(()) -} - -/// Get the schema for a specific item (script or flow) -pub async fn get_item_schema( - path: &str, - user_db: &UserDB, - authed: &ApiAuthed, - workspace_id: &str, - item_type: &str, -) -> Result, ErrorData> { - let mut sqlb = SqlBuilder::select_from(&format!("{} as o", item_type)); - sqlb.fields(&["o.schema"]); - sqlb.and_where("o.path = ?".bind(&path)); - sqlb.and_where("o.workspace_id = ?".bind(&workspace_id)); - sqlb.and_where("o.archived = false"); - sqlb.and_where("o.draft_only IS NOT TRUE"); - let sql = sqlb.sql().map_err(|_e| { - tracing::error!("failed to build sql: {}", _e); - ErrorData::internal_error("failed to build sql", None) - })?; - let mut tx = user_db - .clone() - .begin(authed) - .await - .map_err(|_e| ErrorData::internal_error("failed to begin transaction", None))?; - let item = sqlx::query_as::<_, ItemSchema>(&sql) - .fetch_one(&mut *tx) - .await - .map_err(|_e| { - tracing::error!("failed to fetch item schema: {}", _e); - ErrorData::internal_error("failed to fetch item schema", None) - })?; - tx.commit() - .await - .map_err(|_e| ErrorData::internal_error("failed to commit transaction", None))?; - Ok(item.schema) -} - -/// Get all resource types from the database -pub async fn get_resources_types( - user_db: &UserDB, - authed: &ApiAuthed, - workspace_id: &str, -) -> Result, ErrorData> { - let mut sqlb = SqlBuilder::select_from("resource_type as o"); - sqlb.fields(&["o.name", "o.description"]); - sqlb.and_where("o.workspace_id = ?".bind(&workspace_id)); - let sql = sqlb.sql().map_err(|_e| { - tracing::error!("failed to build sql: {}", _e); - ErrorData::internal_error("failed to build sql", None) - })?; - let mut tx = user_db - .clone() - .begin(authed) - .await - .map_err(|_e| ErrorData::internal_error("failed to begin transaction", None))?; - let rows = sqlx::query_as::<_, ResourceType>(&sql) - .fetch_all(&mut *tx) - .await - .map_err(|_e| { - tracing::error!("Failed to fetch resource types: {}", _e); - ErrorData::internal_error("failed to fetch resource types", None) - })?; - tx.commit() - .await - .map_err(|_e| ErrorData::internal_error("failed to commit transaction", None))?; - Ok(rows) -} - -/// Get resources by type from the database -pub async fn get_resources( - user_db: &UserDB, - authed: &ApiAuthed, - workspace_id: &str, - resource_type: &str, -) -> Result, ErrorData> { - let mut sqlb = SqlBuilder::select_from("resource as o"); - sqlb.fields(&["o.path", "o.description", "o.resource_type"]); - sqlb.and_where("o.workspace_id = ?".bind(&workspace_id)); - sqlb.and_where("o.resource_type = ?".bind(&resource_type)); - let sql = sqlb.sql().map_err(|_e| { - tracing::error!("failed to build sql: {}", _e); - ErrorData::internal_error("failed to build sql", None) - })?; - let mut tx = user_db - .clone() - .begin(authed) - .await - .map_err(|_e| ErrorData::internal_error("failed to begin transaction", None))?; - let rows = sqlx::query_as::<_, ResourceInfo>(&sql) - .fetch_all(&mut *tx) - .await - .map_err(|_e| { - tracing::error!("Failed to fetch resources: {}", _e); - ErrorData::internal_error("failed to fetch resources", None) - })?; - tx.commit() - .await - .map_err(|_e| ErrorData::internal_error("failed to commit transaction", None))?; - - Ok(rows) -} - -/// Generic function to get items (scripts or flows) from the database -pub async fn get_items sqlx::FromRow<'a, sqlx::postgres::PgRow> + Send + Unpin>( - user_db: &UserDB, - authed: &ApiAuthed, - workspace_id: &str, - scope_type: &str, - item_type: &str, -) -> Result, ErrorData> { - let mut sqlb = SqlBuilder::select_from(&format!("{} as o", item_type)); - let fields = vec!["o.path", "o.summary", "o.description", "o.schema"]; - sqlb.fields(&fields); - if scope_type == "favorites" { - sqlb.join("favorite") - .on("favorite.favorite_kind = ? AND favorite.workspace_id = o.workspace_id AND favorite.path = o.path AND favorite.usr = ?".bind(&item_type) - .bind(&authed.username)); - } - sqlb.and_where("o.workspace_id = ?".bind(&workspace_id)) - .and_where("o.archived = false") - .and_where("o.draft_only IS NOT TRUE"); - - if item_type == "script" { - sqlb.and_where("(o.no_main_func IS NOT TRUE OR o.no_main_func IS NULL)"); - } - - sqlb.order_by( - if item_type == "flow" { - "o.edited_at" - } else { - "o.created_at" - }, - false, - ) - .limit(100); - let sql = sqlb.sql().map_err(|_e| { - tracing::error!("failed to build sql: {}", _e); - ErrorData::internal_error("failed to build sql", None) - })?; - let mut tx = user_db - .clone() - .begin(authed) - .await - .map_err(|_e| ErrorData::internal_error("failed to begin transaction", None))?; - let rows = sqlx::query_as::<_, T>(&sql) - .fetch_all(&mut *tx) - .await - .map_err(|_e| { - tracing::error!("Failed to fetch {}: {}", item_type, _e); - ErrorData::internal_error(format!("failed to fetch {}", item_type), None) - })?; - tx.commit() - .await - .map_err(|_e| ErrorData::internal_error("failed to commit transaction", None))?; - Ok(rows) -} - -/// Get scripts from the Hub -pub async fn get_scripts_from_hub( - db: &DB, - scope_integrations: Option<&str>, -) -> Result, ErrorData> { - let query_params = Some(vec![ - ("limit", "100".to_string()), - ("with_schema", "true".to_string()), - ("apps", scope_integrations.unwrap_or("").to_string()), - ]); - let url = format!("{}/scripts/top", *HUB_BASE_URL.read().await); - let (_status_code, _headers, response) = - query_elems_from_hub(&HTTP_CLIENT, &url, query_params, &db) - .await - .map_err(|e| { - tracing::error!("Failed to get items from hub: {}", e); - ErrorData::internal_error(format!("Failed to get items from hub: {}", e), None) - })?; - - use axum::body::to_bytes; - let body_bytes = to_bytes(response, usize::MAX).await.map_err(|e| { - tracing::error!("Failed to read response body: {}", e); - ErrorData::internal_error(format!("Failed to read response body: {}", e), None) - })?; - let body_str = String::from_utf8(body_bytes.to_vec()).map_err(|e| { - tracing::error!("Failed to decode response body: {}", e); - ErrorData::internal_error(format!("Failed to decode response body: {}", e), None) - })?; - let hub_response: HubResponse = serde_json::from_str(&body_str).map_err(|e| { - tracing::error!("Failed to parse hub response: {}", e); - ErrorData::internal_error(format!("Failed to parse hub response: {}", e), None) - })?; - - Ok(hub_response.asks) -} - -/// Get the schema for a Hub script -pub async fn get_hub_script_schema(path: &str, db: &DB) -> Result, ErrorData> { - let strip_path = StripPath(path.to_string()); - let res = get_full_hub_script_by_path(strip_path, &HTTP_CLIENT, Some(db)) - .await - .map_err(|e| { - tracing::error!("Failed to get hub script: {}", e); - ErrorData::internal_error(format!("Failed to get hub script: {}", e), None) - })?; - match serde_json::from_str::(res.schema.get()) { - Ok(schema) => Ok(Some(schema)), - Err(e) => { - tracing::warn!("Failed to convert schema: {}", e); - Ok(None) - } - } -} diff --git a/backend/windmill-api/src/mcp/utils/mod.rs b/backend/windmill-api/src/mcp/utils/mod.rs deleted file mode 100644 index e4072313a0..0000000000 --- a/backend/windmill-api/src/mcp/utils/mod.rs +++ /dev/null @@ -1,10 +0,0 @@ -//! Utility functions and helpers for MCP server -//! -//! This module contains various utility functions for schema transformation, -//! database operations, data models, and path transformations. - -pub mod models; -pub mod database; -pub mod schema; -pub mod transform; -pub mod scope_matcher; \ No newline at end of file diff --git a/backend/windmill-api/src/mcp/utils/schema.rs b/backend/windmill-api/src/mcp/utils/schema.rs deleted file mode 100644 index b441d94a78..0000000000 --- a/backend/windmill-api/src/mcp/utils/schema.rs +++ /dev/null @@ -1,143 +0,0 @@ -//! Schema transformation utilities for MCP server -//! -//! Contains functions for transforming Windmill schemas into MCP-compatible formats, -//! including resource enrichment and schema conversion utilities. - -use windmill_mcp::server::ErrorData; -use serde_json::Value; -use std::collections::HashMap; -use windmill_common::db::UserDB; -use windmill_common::scripts::Schema; - -use super::database::get_resources; -use super::models::{ResourceInfo, ResourceType, SchemaType}; -use super::transform::apply_key_transformation; -use crate::db::ApiAuthed; - -/// Convert a Windmill Schema to a SchemaType -pub fn convert_schema_to_schema_type(schema: Option) -> SchemaType { - let schema_obj = if let Some(ref s) = schema { - match serde_json::from_str::(s.0.get()) { - Ok(val) => val, - Err(_) => SchemaType::default(), - } - } else { - SchemaType::default() - }; - schema_obj -} - -/// Transform the schema for resources by enriching with resource information -pub async fn transform_schema_for_resources( - schema: &SchemaType, - user_db: &UserDB, - authed: &ApiAuthed, - w_id: &str, - resources_cache: &mut HashMap>, - resources_types: &Vec, -) -> Result { - let mut schema_obj: SchemaType = schema.clone(); - - // replace invalid char in property key with underscore - let replacements: Vec<(String, String, Value)> = schema_obj - .properties - .iter() - .filter_map(|(key, value)| { - if key.chars().any(|c| !c.is_alphanumeric() && c != '_') { - let new_key = apply_key_transformation(key); - Some((key.clone(), new_key, value.clone())) - } else { - None - } - }) - .collect(); - - for (old_key, new_key, value) in replacements { - schema_obj.properties.remove(&old_key); - schema_obj.properties.insert(new_key, value); - } - - for (_key, prop_value) in schema_obj.properties.iter_mut() { - if let Value::Object(prop_map) = prop_value { - // if property is a resource, fetch the resource type infos, and add each available resource to the description - if let Some(format_value) = prop_map.get("format") { - if let Value::String(format_str) = format_value { - if format_str.starts_with("resource-") { - let resource_type_key = - format_str.split("-").last().unwrap_or_default().to_string(); - let resource_type = resources_types - .iter() - .find(|rt| rt.name == resource_type_key); - let resource_type_obj = resource_type.cloned(); - - if !resources_cache.contains_key(&resource_type_key) { - let available_resources = - get_resources(user_db, authed, &w_id, &resource_type_key).await; - - match available_resources { - Ok(cache_data) => { - resources_cache.insert(resource_type_key.clone(), cache_data); - } - Err(e) => { - tracing::error!("Failed to fetch resource cache data: {}", e); - continue; // Skip this property if fetching failed - } - } - } - - if let Some(resource_cache) = resources_cache.get(&resource_type_key) { - let resources_count = resource_cache.len(); - let description = match resource_type_obj { - Some(resource_type_obj) => format!( - "This is a resource named `{}` with the following description: `{}`.\\nThe path of the resource should be used to specify the resource.\\n{}", - resource_type_obj.name, - resource_type_obj.description.as_deref().unwrap_or("No description"), - if resources_count == 0 { - "This resource does not have any available instances, you should create one from your windmill workspace." - } else if resources_count > 1 { - "This resource has multiple available instances, you should precisely select the one you want to use." - } else { - "There is 1 resource available." - } - ), - None => "An object parameter.".to_string() - }; - prop_map - .insert("type".to_string(), Value::String("string".to_string())); - prop_map.insert("description".to_string(), Value::String(description)); - if resources_count > 0 { - let resources_description = resource_cache - .iter() - .map(|resource| { - format!( - "{}: $res:{}", - resource.description.as_deref().unwrap_or("No title"), - resource.path - ) - }) - .collect::>() - .join("\\n"); - - prop_map.insert( - "description".to_string(), - Value::String(format!( - "{}\\nHere are the available resources, in the format title:path. Title can be empty. Path should be used to specify the resource:\\n{}", - prop_map.get("description").unwrap_or(&Value::String("No description".to_string())), - resources_description - )), - ); - } - } - } - } - } - } else { - tracing::warn!( - "Schema property value is not a JSON object: {:?}", - prop_value - ); - } - } - - Ok(schema_obj) -} diff --git a/backend/windmill-mcp/Cargo.toml b/backend/windmill-mcp/Cargo.toml index a3e4bfed5f..afe2547f19 100644 --- a/backend/windmill-mcp/Cargo.toml +++ b/backend/windmill-mcp/Cargo.toml @@ -10,7 +10,7 @@ path = "src/lib.rs" [features] default = [] -server = ["rmcp/transport-streamable-http-server", "rmcp/transport-streamable-http-server-session", "rmcp/transport-worker"] +server = ["rmcp/transport-streamable-http-server", "rmcp/transport-streamable-http-server-session", "rmcp/transport-worker", "dep:sqlx", "dep:async-trait", "dep:http", "dep:tokio-util", "dep:tokio"] auth = ["rmcp/auth", "dep:oauth2"] [dependencies] @@ -22,3 +22,9 @@ serde.workspace = true serde_json.workspace = true tracing.workspace = true rmcp.workspace = true +sqlx = { workspace = true, optional = true } +async-trait = { workspace = true, optional = true } +http = { workspace = true, optional = true } +tokio-util = { workspace = true, features = ["rt"], optional = true } +tokio = { workspace = true, optional = true } +futures.workspace = true diff --git a/backend/windmill-mcp/src/client/mod.rs b/backend/windmill-mcp/src/client/mod.rs new file mode 100644 index 0000000000..2364571486 --- /dev/null +++ b/backend/windmill-mcp/src/client/mod.rs @@ -0,0 +1,209 @@ +//! MCP Client implementation +//! +//! This module provides functionality for connecting to external MCP servers +//! and executing tools on them. + +mod types; + +pub use types::{McpResource, McpToolSource}; + +use anyhow::{Context, Result}; +use reqwest::header::{HeaderMap, HeaderName, HeaderValue}; +use rmcp::{ + model::{ + CallToolRequestParam, ClientCapabilities, ClientInfo, Implementation, + InitializeRequestParam, Tool as McpTool, + }, + service::RunningService, + transport::{ + streamable_http_client::StreamableHttpClientTransportConfig, StreamableHttpClientTransport, + }, + RoleClient, ServiceExt, +}; +use serde_json::{json, Value}; +use std::str::FromStr; +use windmill_common::variables::get_secret_value_as_admin; +use windmill_common::DB; + +/// MCP client for communicating with external MCP servers +pub struct McpClient { + /// The underlying rmcp client + client: RunningService, + /// Cached list of available tools from the server + available_tools: Vec, +} + +impl McpClient { + /// Create a new MCP client from a resource configuration + pub async fn from_resource(resource: McpResource, db: &DB, w_id: &str) -> Result { + // Build custom reqwest client with headers if provided + let mut headers = HeaderMap::new(); + if let Some(token_path) = &resource.token { + if !token_path.trim().is_empty() { + let value = + get_secret_value_as_admin(db, w_id, token_path.trim_start_matches("$var:")) + .await?; + headers.insert( + HeaderName::from_static("authorization"), + HeaderValue::from_str(format!("Bearer {}", value).as_str())?, + ); + } + } + if let Some(resource_headers) = &resource.headers { + for (key, value) in resource_headers { + match (HeaderName::from_str(key), HeaderValue::from_str(value)) { + (Ok(name), Ok(value)) => { + headers.insert(name, value); + } + _ => { + tracing::warn!("Invalid header: {}={}", key, value); + } + } + } + } + + let reqwest_client = reqwest::Client::builder() + .default_headers(headers) + .build() + .context("Failed to build HTTP client")?; + + // Create the HTTP transport with custom client + let config = StreamableHttpClientTransportConfig::with_uri(resource.url.as_str()); + let transport = StreamableHttpClientTransport::with_client(reqwest_client, config); + + // Set up client info + let client_info = ClientInfo { + protocol_version: Default::default(), + capabilities: ClientCapabilities::default(), + client_info: Implementation { + name: "windmill-ai-agent".to_string(), + title: Some("Windmill AI Agent".to_string()), + version: env!("CARGO_PKG_VERSION").to_string(), + website_url: None, + icons: None, + }, + }; + + // Initialize the connection + let client = client_info + .serve(transport) + .await + .context("Failed to connect to MCP server")?; + + // Immediately fetch available tools + let available_tools = client + .list_tools(Default::default()) + .await + .context("Failed to list tools from MCP server")? + .tools; + + Ok(Self { client, available_tools }) + } + + /// Get the list of available tools from the MCP server + pub fn available_tools(&self) -> &[McpTool] { + &self.available_tools + } + + /// Call a tool on the MCP server, with openai-style arguments + pub async fn call_tool(&self, name: &str, arguments: &str) -> Result { + // Convert OpenAI-style arguments to MCP format + let mcp_args = + Self::openai_args_to_mcp_args(arguments).context("Failed to parse tool arguments")?; + + let result = self + .client + .call_tool(CallToolRequestParam { + name: name.to_string().into(), + arguments: mcp_args, + task: None, + }) + .await + .context(format!("Failed to call MCP tool: {}", name))?; + + // Convert the result to a JSON value + // MCP tools return ToolResult which contains content array + let result_json = + serde_json::to_value(&result).context("Failed to serialize MCP tool result")?; + + Ok(result_json) + } + + /// Close the connection + pub async fn shutdown(self) -> Result<()> { + self.client.cancel().await?; + Ok(()) + } + + /// Fix array schemas to ensure they have the required 'items' property + /// OpenAI requires all array types to have an 'items' field. MCP servers may + /// return schemas without this field, so we add a default. + pub fn fix_array_schemas(schema: &mut Value) { + if let Value::Object(obj) = schema { + // Check if this is an array type + if let Some(type_val) = obj.get("type") { + let is_array = match type_val { + Value::String(s) => s == "array", + Value::Array(arr) => arr.iter().any(|v| v.as_str() == Some("array")), + _ => false, + }; + + // If it's an array and missing 'items', add a default + if is_array && !obj.contains_key("items") { + obj.insert("items".to_string(), json!({})); + } + } + + // Recursively fix nested schemas + if let Some(Value::Object(props)) = obj.get_mut("properties") { + for value in props.values_mut() { + Self::fix_array_schemas(value); + } + } + + // Fix items if present (for nested arrays) + if let Some(items) = obj.get_mut("items") { + Self::fix_array_schemas(items); + } + + // Fix oneOf, anyOf, allOf schemas + for key in &["oneOf", "anyOf", "allOf"] { + if let Some(Value::Array(schemas)) = obj.get_mut(*key) { + for schema in schemas { + Self::fix_array_schemas(schema); + } + } + } + + // Fix additionalProperties if it's a schema + if let Some(additional) = obj.get_mut("additionalProperties") { + if additional.is_object() { + Self::fix_array_schemas(additional); + } + } + } + } + + /// Convert OpenAI-style tool call arguments to MCP format + /// OpenAI sends arguments as a JSON string, MCP expects a Map + fn openai_args_to_mcp_args( + args_str: &str, + ) -> Result>> { + if args_str.trim().is_empty() { + return Ok(None); + } + + let args_value: serde_json::Value = + serde_json::from_str(args_str).context("Failed to parse tool call arguments")?; + + match args_value { + serde_json::Value::Object(map) => Ok(Some(map)), + serde_json::Value::Null => Ok(None), + _ => Ok(Some( + vec![("value".to_string(), args_value)] + .into_iter() + .collect(), + )), + } + } +} diff --git a/backend/windmill-mcp/src/client/types.rs b/backend/windmill-mcp/src/client/types.rs new file mode 100644 index 0000000000..232f2a5a9a --- /dev/null +++ b/backend/windmill-mcp/src/client/types.rs @@ -0,0 +1,32 @@ +//! Client-specific types for MCP +//! +//! Contains configuration and metadata types used for MCP client connections. + +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + +/// MCP server resource configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct McpResource { + /// Name of the MCP resource (used for prefixing tools) + pub name: String, + /// HTTP URL for the MCP server endpoint + pub url: String, + /// Optional token for authentication + #[serde(skip_serializing_if = "Option::is_none")] + pub token: Option, + /// Optional headers + #[serde(skip_serializing_if = "Option::is_none")] + pub headers: Option>, +} + +/// Metadata for tracking MCP tool sources +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct McpToolSource { + /// Name of the MCP resource this tool comes from + pub name: String, + /// Original tool name in the MCP server + pub tool_name: String, + /// Path of the MCP resource + pub resource_path: String, +} diff --git a/backend/windmill-mcp/src/common/mod.rs b/backend/windmill-mcp/src/common/mod.rs new file mode 100644 index 0000000000..d67c1718cc --- /dev/null +++ b/backend/windmill-mcp/src/common/mod.rs @@ -0,0 +1,16 @@ +//! Common types and utilities for MCP server and client +//! +//! This module contains shared data structures, transformation utilities, +//! and scope parsing functionality used throughout the MCP implementation. + +pub mod schema; +pub mod scope; +pub mod transform; +pub mod types; + +pub use schema::convert_schema_to_schema_type; +pub use scope::{is_resource_allowed, parse_mcp_scopes, McpScopeConfig}; +pub use transform::{ + apply_key_transformation, reverse_transform, reverse_transform_key, transform_path, +}; +pub use types::*; diff --git a/backend/windmill-mcp/src/common/schema.rs b/backend/windmill-mcp/src/common/schema.rs new file mode 100644 index 0000000000..c7cec8849e --- /dev/null +++ b/backend/windmill-mcp/src/common/schema.rs @@ -0,0 +1,41 @@ +//! Schema conversion utilities for MCP server +//! +//! Contains functions for converting Windmill schemas into MCP-compatible formats. + +use std::collections::HashSet; + +use serde_json::Value; + +use super::types::SchemaType; +use windmill_common::scripts::Schema; + +/// Convert a Windmill Schema to a SchemaType +pub fn convert_schema_to_schema_type(schema: Option) -> SchemaType { + let schema_obj = if let Some(ref s) = schema { + match serde_json::from_str::(s.0.get()) { + Ok(val) => val, + Err(_) => SchemaType::default(), + } + } else { + SchemaType::default() + }; + schema_obj +} + +/// Extract resource type keys from a schema +/// +/// Scans the schema properties for fields with format "resource-{type}" +/// and returns a set of all unique resource type names found. +pub fn extract_resource_types_from_schema(schema: &SchemaType) -> HashSet { + let mut resource_types = HashSet::new(); + for (_key, prop_value) in schema.properties.iter() { + if let Value::Object(prop_map) = prop_value { + if let Some(Value::String(format_str)) = prop_map.get("format") { + if let Some(rt) = format_str.strip_prefix("resource-") { + resource_types.insert(rt.to_string()); + } + } + } + } + resource_types +} diff --git a/backend/windmill-api/src/mcp/utils/scope_matcher.rs b/backend/windmill-mcp/src/common/scope.rs similarity index 86% rename from backend/windmill-api/src/mcp/utils/scope_matcher.rs rename to backend/windmill-mcp/src/common/scope.rs index 4b50d747ac..7da4e2cebc 100644 --- a/backend/windmill-api/src/mcp/utils/scope_matcher.rs +++ b/backend/windmill-mcp/src/common/scope.rs @@ -3,8 +3,6 @@ //! Contains utilities for parsing and matching MCP token scopes to determine //! which scripts, flows, and endpoints a token has access to. -use windmill_mcp::server::ErrorData; - /// Configuration for MCP scopes parsed from token scopes #[derive(Debug, Clone, Default)] pub struct McpScopeConfig { @@ -24,8 +22,26 @@ pub struct McpScopeConfig { pub hub_apps: Option, } +impl McpScopeConfig { + /// Check if a resource is allowed based on its type and path + pub fn is_allowed(&self, resource_type: &str, path: &str) -> bool { + if self.all { + return true; + } + + let patterns = match resource_type { + "script" => &self.scripts, + "flow" => &self.flows, + "endpoint" => &self.endpoints, + _ => return false, + }; + + is_resource_allowed(path, patterns) + } +} + /// Parse MCP scopes from token scope strings -pub fn parse_mcp_scopes(scopes: &[String]) -> Result { +pub fn parse_mcp_scopes(scopes: &[String]) -> Result { let mut config = McpScopeConfig::default(); for scope in scopes { @@ -69,19 +85,19 @@ pub fn parse_mcp_scopes(scopes: &[String]) -> Result if let Some(resources) = scope.strip_prefix("mcp:scripts:") { // New granular script scope: mcp:scripts:path1,path2,f/folder/* - config.scripts.extend(parse_resource_list(resources)?); + config.scripts.extend(parse_resource_list(resources)); continue; } if let Some(resources) = scope.strip_prefix("mcp:flows:") { // New granular flow scope: mcp:flows:path1,path2,f/folder/* - config.flows.extend(parse_resource_list(resources)?); + config.flows.extend(parse_resource_list(resources)); continue; } if let Some(resources) = scope.strip_prefix("mcp:endpoints:") { // New granular endpoint scope: mcp:endpoints:name1,name2 - config.endpoints.extend(parse_resource_list(resources)?); + config.endpoints.extend(parse_resource_list(resources)); continue; } @@ -94,16 +110,16 @@ pub fn parse_mcp_scopes(scopes: &[String]) -> Result } /// Parse comma-separated resource list -fn parse_resource_list(resources: &str) -> Result, ErrorData> { +fn parse_resource_list(resources: &str) -> Vec { if resources.is_empty() { - return Ok(vec![]); + return vec![]; } - Ok(resources + resources .split(',') .map(|s| s.trim().to_string()) .filter(|s| !s.is_empty()) - .collect()) + .collect() } /// Check if a resource path matches any pattern in the allowed list @@ -226,4 +242,16 @@ mod tests { let empty: Vec = vec![]; assert!(!is_resource_allowed("any/path", &empty)); } + + #[test] + fn test_scope_config_is_allowed() { + let mut config = McpScopeConfig::default(); + config.scripts.push("u/admin/*".to_string()); + config.flows.push("f/automation/*".to_string()); + + assert!(config.is_allowed("script", "u/admin/test")); + assert!(!config.is_allowed("script", "u/other/test")); + assert!(config.is_allowed("flow", "f/automation/test")); + assert!(!config.is_allowed("flow", "f/other/test")); + } } diff --git a/backend/windmill-api/src/mcp/utils/transform.rs b/backend/windmill-mcp/src/common/transform.rs similarity index 78% rename from backend/windmill-api/src/mcp/utils/transform.rs rename to backend/windmill-mcp/src/common/transform.rs index d4ef52fd3f..9a6fa317b1 100644 --- a/backend/windmill-api/src/mcp/utils/transform.rs +++ b/backend/windmill-mcp/src/common/transform.rs @@ -3,9 +3,9 @@ //! Contains functions for transforming paths, keys, and other identifiers //! to make them compatible with MCP tool naming requirements. -use super::models::SchemaType; +use super::types::SchemaType; -// MCP clients do not allow names longer than 60 characters +/// MCP clients do not allow names longer than 60 characters const MAX_PATH_LENGTH: usize = 60; /// Transform the path for workspace scripts/flows @@ -114,3 +114,38 @@ pub fn reverse_transform_key(transformed_key: &str, schema_obj: &Option, @@ -39,16 +42,13 @@ pub struct SchemaType { impl Default for SchemaType { fn default() -> Self { - Self { - r#type: "object".to_string(), - properties: HashMap::new(), - required: vec![], - } + Self { r#type: "object".to_string(), properties: HashMap::new(), required: vec![] } } } /// Script information from database -#[derive(Serialize, FromRow, Debug)] +#[derive(Serialize, Debug)] +#[cfg_attr(feature = "server", derive(FromRow))] pub struct ScriptInfo { pub path: String, pub summary: Option, @@ -57,7 +57,8 @@ pub struct ScriptInfo { } /// Flow information from database -#[derive(Serialize, FromRow, Debug)] +#[derive(Serialize, Debug)] +#[cfg_attr(feature = "server", derive(FromRow))] pub struct FlowInfo { pub path: String, pub summary: Option, @@ -66,7 +67,8 @@ pub struct FlowInfo { } /// Resource information from database -#[derive(Serialize, FromRow, Debug, Clone)] +#[derive(Serialize, Debug, Clone)] +#[cfg_attr(feature = "server", derive(FromRow))] pub struct ResourceInfo { pub path: String, pub description: Option, @@ -74,25 +76,34 @@ pub struct ResourceInfo { } /// Resource type information from database -#[derive(Serialize, FromRow, Debug, Clone)] +#[derive(Serialize, Debug, Clone)] +#[cfg_attr(feature = "server", derive(FromRow))] pub struct ResourceType { pub name: String, pub description: Option, } /// Schema holder for database queries -#[derive(Serialize, FromRow)] +#[derive(Serialize)] +#[cfg_attr(feature = "server", derive(FromRow))] pub struct ItemSchema { pub schema: Option, } /// Trait for objects that can be converted to MCP tools pub trait ToolableItem { + /// Get the path or identifier for this item (transformed for MCP compatibility) fn get_path_or_id(&self) -> String; + /// Get the summary/title of this item fn get_summary(&self) -> &str; + /// Get the description of this item fn get_description(&self) -> &str; + /// Get the JSON schema for this item's parameters fn get_schema(&self) -> SchemaType; + /// Whether this item is from the Hub fn is_hub(&self) -> bool; + /// Get the type of this item ("script" or "flow") fn item_type(&self) -> &'static str; + /// Get the integration type (for hub scripts) fn get_integration_type(&self) -> Option; -} \ No newline at end of file +} diff --git a/backend/windmill-mcp/src/lib.rs b/backend/windmill-mcp/src/lib.rs index bf05dc7c45..9026837493 100644 --- a/backend/windmill-mcp/src/lib.rs +++ b/backend/windmill-mcp/src/lib.rs @@ -1,51 +1,33 @@ -/* - * Author: Ruben Fiszel - * Copyright: Windmill Labs, Inc 2022 - * This file and its contents are licensed under the AGPLv3 License. - * Please see the included NOTICE for copyright information and - * LICENSE-AGPL for a copy of the license. - */ +//! Windmill MCP (Model Context Protocol) implementation +//! +//! This crate provides: +//! - MCP client for connecting to external MCP servers (used by AI agents) +//! - Common types and utilities for MCP implementations +//! - MCP server types (when `server` feature is enabled) +//! - OAuth support (when `auth` feature is enabled) -use anyhow::{Context, Result}; -use reqwest::header::{HeaderMap, HeaderName, HeaderValue}; -use serde_json::{json, Value}; -use std::str::FromStr; -use windmill_common::variables::get_secret_value_as_admin; -use windmill_common::DB; +// Common types and utilities module +pub mod common; + +// Client module +pub mod client; + +// Re-export common types at crate root for convenience +pub use common::{ + convert_schema_to_schema_type, is_resource_allowed, parse_mcp_scopes, transform_path, FlowInfo, + HubResponse, HubScriptInfo, ItemSchema, McpScopeConfig, ResourceInfo, ResourceType, SchemaType, + ScriptInfo, ToolableItem, WorkspaceId, +}; + +// Re-export client types at crate root for backward compatibility +pub use client::{McpClient, McpResource, McpToolSource}; // Re-export rmcp types for client usage pub use rmcp::model::Tool as McpTool; -use rmcp::{ - model::{ - CallToolRequestParam, ClientCapabilities, ClientInfo, Implementation, - InitializeRequestParam, - }, - service::RunningService, - transport::{ - streamable_http_client::StreamableHttpClientTransportConfig, StreamableHttpClientTransport, - }, - RoleClient, ServiceExt, -}; -// Re-export rmcp server types when server feature is enabled +// Server module (when server feature is enabled) #[cfg(feature = "server")] -pub mod server { - //! Re-exports of rmcp server types for MCP server implementations - - pub use rmcp::handler::server::ServerHandler; - pub use rmcp::model::{ - Annotated, CallToolRequestParam, CallToolResult, Content, Implementation, - InitializeRequestParam, InitializeResult, ListPromptsResult, ListResourceTemplatesResult, - ListResourcesResult, ListToolsResult, PaginatedRequestParam, ProtocolVersion, RawContent, - RawTextContent, ServerCapabilities, ServerInfo, Tool, ToolAnnotations, - }; - pub use rmcp::service::{RequestContext, RoleServer}; - pub use rmcp::transport::streamable_http_server::{ - session::local::LocalSessionManager, StreamableHttpService, - }; - pub use rmcp::transport::StreamableHttpServerConfig; - pub use rmcp::ErrorData; -} +pub mod server; // Re-export rmcp auth types when auth feature is enabled #[cfg(feature = "auth")] @@ -60,212 +42,3 @@ pub mod oauth { RedirectUrl, Scope, TokenUrl, }; } - -use std::collections::HashMap; - -use serde::{Deserialize, Serialize}; - -/// MCP server resource configuration -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct McpResource { - /// Name of the MCP resource (used for prefixing tools) - pub name: String, - /// HTTP URL for the MCP server endpoint - pub url: String, - /// Optional token for authentication - #[serde(skip_serializing_if = "Option::is_none")] - pub token: Option, - /// Optional headers - #[serde(skip_serializing_if = "Option::is_none")] - pub headers: Option>, -} - -/// Metadata for tracking MCP tool sources -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct McpToolSource { - /// Name of the MCP resource this tool comes from - pub name: String, - /// Original tool name in the MCP server - pub tool_name: String, - /// Path of the MCP resource - pub resource_path: String, -} - -/// MCP client for communicating with external MCP servers -pub struct McpClient { - /// The underlying rmcp client - client: RunningService, - /// Cached list of available tools from the server - available_tools: Vec, -} - -impl McpClient { - /// Create a new MCP client from a resource configuration - pub async fn from_resource(resource: McpResource, db: &DB, w_id: &str) -> Result { - // Build custom reqwest client with headers if provided - let mut headers = HeaderMap::new(); - if let Some(token_path) = &resource.token { - if !token_path.trim().is_empty() { - let value = - get_secret_value_as_admin(db, w_id, token_path.trim_start_matches("$var:")) - .await?; - headers.insert( - HeaderName::from_static("authorization"), - HeaderValue::from_str(format!("Bearer {}", value).as_str())?, - ); - } - } - if let Some(resource_headers) = &resource.headers { - for (key, value) in resource_headers { - match (HeaderName::from_str(key), HeaderValue::from_str(value)) { - (Ok(name), Ok(value)) => { - headers.insert(name, value); - } - _ => { - tracing::warn!("Invalid header: {}={}", key, value); - } - } - } - } - - let reqwest_client = reqwest::Client::builder() - .default_headers(headers) - .build() - .context("Failed to build HTTP client")?; - - // Create the HTTP transport with custom client - let config = StreamableHttpClientTransportConfig::with_uri(resource.url.as_str()); - let transport = StreamableHttpClientTransport::with_client(reqwest_client, config); - - // Set up client info - let client_info = ClientInfo { - protocol_version: Default::default(), - capabilities: ClientCapabilities::default(), - client_info: Implementation { - name: "windmill-ai-agent".to_string(), - title: Some("Windmill AI Agent".to_string()), - version: env!("CARGO_PKG_VERSION").to_string(), - website_url: None, - icons: None, - }, - }; - - // Initialize the connection - let client = client_info - .serve(transport) - .await - .context("Failed to connect to MCP server")?; - - // Immediately fetch available tools - let available_tools = client - .list_tools(Default::default()) - .await - .context("Failed to list tools from MCP server")? - .tools; - - Ok(Self { client, available_tools }) - } - - /// Get the list of available tools from the MCP server - pub fn available_tools(&self) -> &[McpTool] { - &self.available_tools - } - - /// Call a tool on the MCP server, with openai-style arguments - pub async fn call_tool(&self, name: &str, arguments: &str) -> Result { - // Convert OpenAI-style arguments to MCP format - let mcp_args = - Self::openai_args_to_mcp_args(arguments).context("Failed to parse tool arguments")?; - - let result = self - .client - .call_tool(CallToolRequestParam { name: name.to_string().into(), arguments: mcp_args, task: None }) - .await - .context(format!("Failed to call MCP tool: {}", name))?; - - // Convert the result to a JSON value - // MCP tools return ToolResult which contains content array - let result_json = - serde_json::to_value(&result).context("Failed to serialize MCP tool result")?; - - Ok(result_json) - } - - /// Close the connection - pub async fn shutdown(self) -> Result<()> { - self.client.cancel().await?; - Ok(()) - } - - /// Fix array schemas to ensure they have the required 'items' property - /// OpenAI requires all array types to have an 'items' field. MCP servers may - /// return schemas without this field, so we add a default. - pub fn fix_array_schemas(schema: &mut Value) { - if let Value::Object(obj) = schema { - // Check if this is an array type - if let Some(type_val) = obj.get("type") { - let is_array = match type_val { - Value::String(s) => s == "array", - Value::Array(arr) => arr.iter().any(|v| v.as_str() == Some("array")), - _ => false, - }; - - // If it's an array and missing 'items', add a default - if is_array && !obj.contains_key("items") { - obj.insert("items".to_string(), json!({})); - } - } - - // Recursively fix nested schemas - if let Some(Value::Object(props)) = obj.get_mut("properties") { - for value in props.values_mut() { - Self::fix_array_schemas(value); - } - } - - // Fix items if present (for nested arrays) - if let Some(items) = obj.get_mut("items") { - Self::fix_array_schemas(items); - } - - // Fix oneOf, anyOf, allOf schemas - for key in &["oneOf", "anyOf", "allOf"] { - if let Some(Value::Array(schemas)) = obj.get_mut(*key) { - for schema in schemas { - Self::fix_array_schemas(schema); - } - } - } - - // Fix additionalProperties if it's a schema - if let Some(additional) = obj.get_mut("additionalProperties") { - if additional.is_object() { - Self::fix_array_schemas(additional); - } - } - } - } - - /// Convert OpenAI-style tool call arguments to MCP format - /// OpenAI sends arguments as a JSON string, MCP expects a Map - fn openai_args_to_mcp_args( - args_str: &str, - ) -> Result>> { - if args_str.trim().is_empty() { - return Ok(None); - } - - let args_value: serde_json::Value = - serde_json::from_str(args_str).context("Failed to parse tool call arguments")?; - - match args_value { - serde_json::Value::Object(map) => Ok(Some(map)), - serde_json::Value::Null => Ok(None), - _ => Ok(Some( - vec![("value".to_string(), args_value)] - .into_iter() - .collect(), - )), - } - } -} diff --git a/backend/windmill-mcp/src/server/backend.rs b/backend/windmill-mcp/src/server/backend.rs new file mode 100644 index 0000000000..fd01b2b26c --- /dev/null +++ b/backend/windmill-mcp/src/server/backend.rs @@ -0,0 +1,159 @@ +//! MCP Backend trait definitions +//! +//! This module defines the traits that must be implemented by the backend +//! (typically windmill-api) to provide the actual functionality for the MCP server. + +use async_trait::async_trait; +use rmcp::ErrorData; +use serde_json::Value; +use std::collections::HashMap; + +use crate::common::types::{ + FlowInfo, HubScriptInfo, ResourceInfo, ResourceType, SchemaType, ScriptInfo, +}; +use crate::server::endpoints::EndpointTool; + +/// Result type for backend operations using rmcp's ErrorData directly +pub type BackendResult = Result; + +/// Authentication context required by the MCP server +pub trait McpAuth: Send + Sync + Clone + 'static { + /// Get the username + fn username(&self) -> &str; + /// Get the email + fn email(&self) -> &str; + /// Check if user is admin + fn is_admin(&self) -> bool; + /// Check if user is operator + fn is_operator(&self) -> bool; + /// Get user's groups + fn groups(&self) -> &[String]; + /// Get user's folders as (name, can_write, is_owner) + fn folders(&self) -> &[(String, bool, bool)]; + /// Get token scopes + fn scopes(&self) -> Option<&[String]>; + + /// Check if the user has an MCP scope + fn has_mcp_scope(&self) -> bool { + self.scopes() + .map(|s| s.iter().any(|scope| scope.starts_with("mcp:"))) + .unwrap_or(false) + } +} + +/// The core backend trait that windmill-api implements +/// +/// This trait abstracts the windmill-api specific operations needed by the MCP server. +/// By implementing this trait, windmill-api can inject its database access, job execution, +/// and other functionality without windmill-mcp needing to depend on windmill-api directly. +#[async_trait] +pub trait McpBackend: Send + Sync + Clone + 'static { + /// The authentication context type + type Auth: McpAuth; + + // ───────────────────────────────────────────────────────────────── + // Listing Operations + // ───────────────────────────────────────────────────────────────── + + /// List scripts, optionally filtered to favorites only + async fn list_scripts( + &self, + auth: &Self::Auth, + workspace_id: &str, + favorites_only: bool, + ) -> BackendResult>; + + /// List flows, optionally filtered to favorites only + async fn list_flows( + &self, + auth: &Self::Auth, + workspace_id: &str, + favorites_only: bool, + ) -> BackendResult>; + + /// List resource types in workspace + async fn list_resource_types( + &self, + auth: &Self::Auth, + workspace_id: &str, + ) -> BackendResult>; + + /// List resources of a specific type + async fn list_resources( + &self, + auth: &Self::Auth, + workspace_id: &str, + resource_type: &str, + ) -> BackendResult>; + + /// List hub scripts, optionally filtered by app integrations + async fn list_hub_scripts(&self, app_filter: Option<&str>) + -> BackendResult>; + + // ───────────────────────────────────────────────────────────────── + // Schema Operations + // ───────────────────────────────────────────────────────────────── + + /// Get schema for a script or flow + async fn get_item_schema( + &self, + auth: &Self::Auth, + workspace_id: &str, + path: &str, + item_type: &str, + ) -> BackendResult>; + + /// Get schema for a hub script + async fn get_hub_script_schema(&self, path: &str) -> BackendResult>; + + // ───────────────────────────────────────────────────────────────── + // Schema Transformation (requires DB access for resources) + // ───────────────────────────────────────────────────────────────── + + /// Transform schema for resources by enriching with available resource information. + /// The resources_cache should be pre-populated with all needed resource types. + fn transform_schema_for_resources( + &self, + schema: &SchemaType, + resources_cache: &HashMap>, + resources_types: &[ResourceType], + ) -> SchemaType; + + // ───────────────────────────────────────────────────────────────── + // Execution Operations + // ───────────────────────────────────────────────────────────────── + + /// Run a script and wait for result + async fn run_script( + &self, + auth: &Self::Auth, + workspace_id: &str, + path: &str, + args: Value, + ) -> BackendResult; + + /// Run a flow and wait for result + async fn run_flow( + &self, + auth: &Self::Auth, + workspace_id: &str, + path: &str, + args: Value, + ) -> BackendResult; + + /// Call an endpoint tool (generated API endpoint) + async fn call_endpoint( + &self, + auth: &Self::Auth, + workspace_id: &str, + endpoint_tool: &EndpointTool, + args: Value, + ) -> BackendResult; + + // ───────────────────────────────────────────────────────────────── + // Endpoint Tools + // ───────────────────────────────────────────────────────────────── + + /// Get all available endpoint tools + fn all_endpoint_tools(&self) -> Vec; +} diff --git a/backend/windmill-mcp/src/server/endpoints.rs b/backend/windmill-mcp/src/server/endpoints.rs new file mode 100644 index 0000000000..ddf531bd1e --- /dev/null +++ b/backend/windmill-mcp/src/server/endpoints.rs @@ -0,0 +1,103 @@ +//! Endpoint tools for MCP server +//! +//! Contains the EndpointTool structure and utilities for converting +//! them to MCP tools. + +use rmcp::model::{Tool, ToolAnnotations}; +use serde::{Deserialize, Serialize}; +use std::borrow::Cow; +use std::sync::Arc; + +/// Represents an auto-generated endpoint tool from OpenAPI specification +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct EndpointTool { + pub name: Cow<'static, str>, + pub description: Cow<'static, str>, + pub instructions: Cow<'static, str>, + pub path: Cow<'static, str>, + pub method: Cow<'static, str>, + pub path_params_schema: Option, + pub query_params_schema: Option, + pub body_schema: Option, +} + +/// Convert a single endpoint tool to MCP tool +pub fn endpoint_tool_to_mcp_tool(tool: &EndpointTool) -> Tool { + let mut combined_properties = serde_json::Map::new(); + let mut combined_required = Vec::new(); + + // Combine all parameter schemas + let schemas = [ + &tool.path_params_schema, + &tool.query_params_schema, + &tool.body_schema, + ]; + + for schema in schemas.iter().filter_map(|s| s.as_ref()) { + merge_schema_into(&mut combined_properties, &mut combined_required, schema); + } + + let combined_schema = serde_json::json!({ + "type": "object", + "properties": combined_properties, + "required": combined_required + }); + + let description = format!("{}. {}", tool.description, tool.instructions); + + // Create annotations based on HTTP method and endpoint characteristics + let annotations = create_endpoint_annotations(tool); + + Tool { + name: tool.name.clone(), + description: Some(description.into()), + input_schema: Arc::new(combined_schema.as_object().unwrap().clone()), + title: Some(tool.name.to_string()), + output_schema: None, + icons: None, + annotations: Some(annotations), + meta: None, + } +} + +/// Create appropriate annotations for endpoint tools based on HTTP method +fn create_endpoint_annotations(tool: &EndpointTool) -> ToolAnnotations { + let method = tool.method.as_ref(); + + // Determine characteristics based on HTTP method + let (read_only, destructive, idempotent, open_world) = match method { + "GET" => (true, false, true, true), // Read-only, safe, idempotent + "POST" => (false, true, false, true), // Can modify, potentially destructive, not idempotent + "PUT" => (false, false, true, true), // Can modify, typically idempotent updates + "DELETE" => (false, true, true, true), // Destructive but idempotent + "PATCH" => (false, false, false, true), // Partial updates, not guaranteed idempotent + _ => (false, true, false, true), // Default: assume can modify and be destructive + }; + + ToolAnnotations { + title: Some(format!("{} {}", method, tool.path)), + read_only_hint: Some(read_only), + destructive_hint: Some(destructive), + idempotent_hint: Some(idempotent), + open_world_hint: Some(open_world), + } +} + +/// Merge schema into combined properties and required fields +fn merge_schema_into( + combined_properties: &mut serde_json::Map, + combined_required: &mut Vec, + schema: &serde_json::Value, +) { + if let Some(props) = schema.get("properties").and_then(|p| p.as_object()) { + for (key, value) in props { + combined_properties.insert(key.clone(), value.clone()); + } + } + + if let Some(required) = schema.get("required").and_then(|r| r.as_array()) { + for req in required.iter().filter_map(|r| r.as_str()) { + combined_required.push(req.to_string()); + } + } +} diff --git a/backend/windmill-mcp/src/server/mod.rs b/backend/windmill-mcp/src/server/mod.rs new file mode 100644 index 0000000000..90e6083435 --- /dev/null +++ b/backend/windmill-mcp/src/server/mod.rs @@ -0,0 +1,32 @@ +//! MCP Server module +//! +//! This module provides the MCP server implementation including: +//! - `McpBackend` trait for backend implementations +//! - `Runner` struct that implements the MCP protocol +//! - Re-exports of rmcp types + +pub mod backend; +pub mod endpoints; +pub mod runner; +pub mod tools; + +// Re-export main types +pub use backend::{BackendResult, McpAuth, McpBackend}; +pub use endpoints::{endpoint_tool_to_mcp_tool, EndpointTool}; +pub use runner::Runner; +pub use tools::create_tool_from_item; + +// Re-export rmcp types for convenience +pub use rmcp::handler::server::ServerHandler; +pub use rmcp::model::{ + Annotated, CallToolRequestParam, CallToolResult, Content, Implementation, + InitializeRequestParam, InitializeResult, ListPromptsResult, ListResourceTemplatesResult, + ListResourcesResult, ListToolsResult, PaginatedRequestParam, ProtocolVersion, RawContent, + RawTextContent, ServerCapabilities, ServerInfo, Tool, ToolAnnotations, +}; +pub use rmcp::service::{RequestContext, RoleServer}; +pub use rmcp::transport::streamable_http_server::{ + session::local::LocalSessionManager, StreamableHttpService, +}; +pub use rmcp::transport::StreamableHttpServerConfig; +pub use rmcp::ErrorData; diff --git a/backend/windmill-mcp/src/server/runner.rs b/backend/windmill-mcp/src/server/runner.rs new file mode 100644 index 0000000000..39fbda5c7a --- /dev/null +++ b/backend/windmill-mcp/src/server/runner.rs @@ -0,0 +1,373 @@ +//! MCP Server Runner implementation +//! +//! Contains the generic Runner that implements the MCP ServerHandler trait +//! and delegates to a McpBackend for actual functionality. + +use crate::common::schema::extract_resource_types_from_schema; +use crate::common::scope::parse_mcp_scopes; +use crate::common::transform::{reverse_transform, reverse_transform_key}; +use crate::common::types::{ResourceInfo, ToolableItem, WorkspaceId}; +use crate::server::backend::{McpAuth, McpBackend}; +use crate::server::endpoints::endpoint_tool_to_mcp_tool; +use crate::server::tools::create_tool_from_item; +use rmcp::handler::server::ServerHandler; +use rmcp::model::{ + CallToolRequestParam, CallToolResult, Content, Implementation, InitializeRequestParam, + InitializeResult, ListPromptsResult, ListResourceTemplatesResult, ListResourcesResult, + ListToolsResult, PaginatedRequestParam, ProtocolVersion, ServerCapabilities, ServerInfo, +}; +use rmcp::service::{RequestContext, RoleServer}; +use rmcp::ErrorData; +use serde_json::Value; +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; + +// Re-export from http crate for extracting request parts +use http::request::Parts as HttpParts; + +/// MCP Server Runner - generic over the backend implementation +/// +/// This struct implements the MCP ServerHandler trait and uses a McpBackend +/// to perform the actual operations (database queries, job execution, etc.) +pub struct Runner { + backend: Arc, +} + +impl Clone for Runner { + fn clone(&self) -> Self { + Self { backend: self.backend.clone() } + } +} + +impl Runner { + /// Create a new Runner with the given backend + pub fn new(backend: B) -> Self { + Self { backend: Arc::new(backend) } + } + + /// Extract authentication and workspace from request context + fn extract_context( + context: &RequestContext, + ) -> Result<(B::Auth, String), ErrorData> { + let http_parts = context.extensions.get::().ok_or_else(|| { + tracing::error!("http::request::Parts not found"); + ErrorData::internal_error("http::request::Parts not found", None) + })?; + + let auth = http_parts.extensions.get::().ok_or_else(|| { + tracing::error!("Auth extension not found"); + ErrorData::internal_error("Auth extension not found", None) + })?; + + let workspace_id = http_parts + .extensions + .get::() + .ok_or_else(|| { + tracing::error!("WorkspaceId not found"); + ErrorData::internal_error("WorkspaceId not found", None) + }) + .map(|w_id| w_id.0.clone())?; + + // Validate MCP scope + if !auth.has_mcp_scope() { + tracing::error!("Unauthorized: missing mcp scope"); + return Err(ErrorData::internal_error( + "Unauthorized: missing mcp scope", + None, + )); + } + + Ok((auth.clone(), workspace_id)) + } +} + +impl ServerHandler for Runner { + fn get_info(&self) -> ServerInfo { + ServerInfo { + protocol_version: ProtocolVersion::default(), + capabilities: ServerCapabilities::builder().enable_tools().build(), + server_info: Implementation::from_build_env(), + instructions: Some( + "This server provides a list of scripts and flows the user can run on Windmill. \ + Each flow and script is a tool callable with their respective arguments." + .to_string(), + ), + } + } + + async fn initialize( + &self, + _request: InitializeRequestParam, + _context: RequestContext, + ) -> Result { + Ok(self.get_info()) + } + + async fn list_tools( + &self, + _request: Option, + context: RequestContext, + ) -> Result { + let (auth, workspace_id) = Self::extract_context(&context)?; + + // Parse MCP scopes to determine what to expose + let scopes = auth.scopes().unwrap_or(&[]); + let scope_config = + parse_mcp_scopes(scopes).map_err(|e| ErrorData::internal_error(e, None))?; + + let favorites_only = scope_config.favorites; + + // Fetch all items concurrently + let (scripts, flows, resource_types, hub_scripts) = tokio::try_join!( + self.backend + .list_scripts(&auth, &workspace_id, favorites_only), + self.backend + .list_flows(&auth, &workspace_id, favorites_only), + self.backend.list_resource_types(&auth, &workspace_id), + async { + if let Some(ref apps) = scope_config.hub_apps { + self.backend.list_hub_scripts(Some(apps)).await + } else { + Ok(vec![]) + } + } + )?; + + // Filter items based on scope + let filtered_scripts: Vec<_> = scripts + .into_iter() + .filter(|s| !scope_config.granular || scope_config.is_allowed("script", &s.path)) + .collect(); + + let filtered_flows: Vec<_> = flows + .into_iter() + .filter(|f| !scope_config.granular || scope_config.is_allowed("flow", &f.path)) + .collect(); + + // Collect all needed resource types from all schemas + let mut needed_resource_types: HashSet = HashSet::new(); + for script in &filtered_scripts { + needed_resource_types.extend(extract_resource_types_from_schema(&script.get_schema())); + } + for flow in &filtered_flows { + needed_resource_types.extend(extract_resource_types_from_schema(&flow.get_schema())); + } + for hub_script in &hub_scripts { + needed_resource_types + .extend(extract_resource_types_from_schema(&hub_script.get_schema())); + } + + // Pre-fetch all resources + let resource_futures: Vec<_> = needed_resource_types + .into_iter() + .map(|rt| { + let backend = self.backend.clone(); + let auth = auth.clone(); + let workspace_id = workspace_id.clone(); + async move { + backend + .list_resources(&auth, &workspace_id, &rt) + .await + .map(|resources| (rt, resources)) + } + }) + .collect(); + + let resource_results = futures::future::try_join_all(resource_futures).await?; + let resources_cache: HashMap> = + resource_results.into_iter().collect(); + + let mut tools = Vec::new(); + + for script in &filtered_scripts { + tools.push(create_tool_from_item( + script, + self.backend.as_ref(), + &resources_cache, + &resource_types, + )); + } + + for flow in &filtered_flows { + tools.push(create_tool_from_item( + flow, + self.backend.as_ref(), + &resources_cache, + &resource_types, + )); + } + + for hub_script in &hub_scripts { + tools.push(create_tool_from_item( + hub_script, + self.backend.as_ref(), + &resources_cache, + &resource_types, + )); + } + + // Add endpoint tools from the generated MCP tools, filtered by scope + let endpoint_tools = self.backend.all_endpoint_tools(); + for endpoint_tool in endpoint_tools { + if scope_config.granular && !scope_config.is_allowed("endpoint", &endpoint_tool.name) { + continue; + } + + tools.push(endpoint_tool_to_mcp_tool(&endpoint_tool)); + } + + Ok(ListToolsResult { tools, next_cursor: None, meta: None }) + } + + async fn call_tool( + &self, + request: CallToolRequestParam, + context: RequestContext, + ) -> Result { + let (auth, workspace_id) = Self::extract_context(&context)?; + + // Parse MCP scopes for authorization + let scopes = auth.scopes().unwrap_or(&[]); + let scope_config = + parse_mcp_scopes(scopes).map_err(|e| ErrorData::internal_error(e, None))?; + + // Handle truncated tool names + if request.name.ends_with("_TRUNC") { + return Ok(CallToolResult::error(vec![rmcp::model::Annotated::new( + rmcp::model::RawContent::Text(rmcp::model::RawTextContent { + text: "Tool path is too long. Consider shortening it to make it compatible with MCP.".to_string(), + meta: None, + }), + None, + )])); + } + + let args = request.arguments.map(Value::Object).unwrap_or(Value::Null); + + // Check if this is an endpoint tool + let endpoint_tools = self.backend.all_endpoint_tools(); + for endpoint_tool in &endpoint_tools { + if endpoint_tool.name.as_ref() == request.name { + // Validate endpoint scope + if scope_config.granular + && !scope_config.is_allowed("endpoint", &endpoint_tool.name) + { + return Err(ErrorData::internal_error( + format!( + "Access denied: endpoint '{}' not in token scope", + endpoint_tool.name + ), + None, + )); + } + + // This is an endpoint tool, call via backend + let result = self + .backend + .call_endpoint(&auth, &workspace_id, endpoint_tool, args) + .await + .map_err(|e| ErrorData::internal_error(e.message, None))?; + + return Ok(CallToolResult::success(vec![Content::text( + serde_json::to_string_pretty(&result).unwrap_or_else(|_| "{}".to_string()), + )])); + } + } + + // Not an endpoint tool - parse as script/flow + let (tool_type, path, is_hub) = reverse_transform(&request.name).map_err(|e| { + ErrorData::internal_error(format!("Failed to parse tool name: {}", e), None) + })?; + + // Validate script/flow scope + if !is_hub && scope_config.granular { + if tool_type == "script" && !scope_config.is_allowed("script", &path) { + return Err(ErrorData::internal_error( + format!("Access denied: script '{}' not in token scope", path), + None, + )); + } else if tool_type == "flow" && !scope_config.is_allowed("flow", &path) { + return Err(ErrorData::internal_error( + format!("Access denied: flow '{}' not in token scope", path), + None, + )); + } + } + + // Get item schema for argument transformation + let item_schema = if is_hub { + self.backend + .get_hub_script_schema(&format!("hub/{}", path)) + .await + .map_err(|e| ErrorData::internal_error(e.message, None))? + } else { + self.backend + .get_item_schema(&auth, &workspace_id, &path, tool_type) + .await + .map_err(|e| ErrorData::internal_error(e.message, None))? + }; + + // Transform arguments back to original key names + let transformed_args = if let Value::Object(map) = args { + let mut args_hash = HashMap::new(); + for (k, v) in map { + let original_key = reverse_transform_key(&k, &item_schema); + args_hash.insert(original_key, v); + } + Value::Object(args_hash.into_iter().collect()) + } else { + args + }; + + let script_or_flow_path = if is_hub { + format!("hub/{}", path) + } else { + path + }; + + // Execute script or flow + let result = if tool_type == "script" { + self.backend + .run_script(&auth, &workspace_id, &script_or_flow_path, transformed_args) + .await + } else { + self.backend + .run_flow(&auth, &workspace_id, &script_or_flow_path, transformed_args) + .await + }; + + match result { + Ok(value) => Ok(CallToolResult::success(vec![Content::text( + serde_json::to_string_pretty(&value).unwrap_or_else(|_| "{}".to_string()), + )])), + Err(e) => Err(ErrorData::internal_error( + format!("Failed to run {}: {}", tool_type, e.message), + None, + )), + } + } + + async fn list_resources( + &self, + _request: Option, + _context: RequestContext, + ) -> Result { + Ok(ListResourcesResult { resources: vec![], next_cursor: None, meta: None }) + } + + async fn list_prompts( + &self, + _request: Option, + _context: RequestContext, + ) -> Result { + Ok(ListPromptsResult::default()) + } + + async fn list_resource_templates( + &self, + _request: Option, + _context: RequestContext, + ) -> Result { + Ok(ListResourceTemplatesResult::default()) + } +} diff --git a/backend/windmill-mcp/src/server/tools.rs b/backend/windmill-mcp/src/server/tools.rs new file mode 100644 index 0000000000..fea875e498 --- /dev/null +++ b/backend/windmill-mcp/src/server/tools.rs @@ -0,0 +1,184 @@ +//! Tool creation utilities for MCP server +//! +//! Contains functionality for converting Windmill items (scripts, flows, hub scripts) +//! into MCP tools. + +use rmcp::model::{Tool, ToolAnnotations}; +use std::borrow::Cow; +use std::collections::HashMap; +use std::sync::Arc; + +use crate::common::schema::convert_schema_to_schema_type; +use crate::common::transform::transform_path; +use crate::common::types::{ + FlowInfo, HubScriptInfo, ResourceInfo, ResourceType, SchemaType, ScriptInfo, ToolableItem, +}; +use crate::server::backend::McpBackend; + +/// Implementation of ToolableItem for ScriptInfo +impl ToolableItem for ScriptInfo { + fn get_path_or_id(&self) -> String { + transform_path(&self.path, "script") + } + + fn get_summary(&self) -> &str { + self.summary.as_deref().unwrap_or("No summary") + } + + fn get_description(&self) -> &str { + self.description.as_deref().unwrap_or("No description") + } + + fn get_schema(&self) -> SchemaType { + convert_schema_to_schema_type(self.schema.clone()) + } + + fn is_hub(&self) -> bool { + false + } + + fn item_type(&self) -> &'static str { + "script" + } + + fn get_integration_type(&self) -> Option { + None + } +} + +/// Implementation of ToolableItem for FlowInfo +impl ToolableItem for FlowInfo { + fn get_path_or_id(&self) -> String { + transform_path(&self.path, "flow") + } + + fn get_summary(&self) -> &str { + self.summary.as_deref().unwrap_or("No summary") + } + + fn get_description(&self) -> &str { + self.description.as_deref().unwrap_or("No description") + } + + fn get_schema(&self) -> SchemaType { + convert_schema_to_schema_type(self.schema.clone()) + } + + fn is_hub(&self) -> bool { + false + } + + fn item_type(&self) -> &'static str { + "flow" + } + + fn get_integration_type(&self) -> Option { + None + } +} + +/// Implementation of ToolableItem for HubScriptInfo +impl ToolableItem for HubScriptInfo { + fn get_path_or_id(&self) -> String { + let id = self.version_id; + let summary = self.summary.as_deref().unwrap_or("No summary"); + format!("hs-{}-{}", id, summary.replace(" ", "_")) + } + + fn get_summary(&self) -> &str { + self.summary.as_deref().unwrap_or("No summary") + } + + fn get_description(&self) -> &str { + self.description.as_deref().unwrap_or("No description") + } + + fn get_schema(&self) -> SchemaType { + match serde_json::from_value::(self.schema.clone().unwrap_or_default()) { + Ok(schema_type) => schema_type, + Err(_) => SchemaType::default(), + } + } + + fn is_hub(&self) -> bool { + true + } + + fn item_type(&self) -> &'static str { + "script" + } + + fn get_integration_type(&self) -> Option { + self.app.clone() + } +} + +/// Create an MCP Tool from a ToolableItem +/// +/// The resources_cache should be pre-populated with all resource types +/// that may be referenced by the item's schema. +pub fn create_tool_from_item( + item: &T, + backend: &B, + resources_cache: &HashMap>, + resources_types: &[ResourceType], +) -> Tool { + let is_hub = item.is_hub(); + let path = item.get_path_or_id(); + let item_type = item.item_type(); + let description = format!( + "This is a {} named `{}` with the following description: `{}`.{}", + item_type, + item.get_summary(), + item.get_description(), + if is_hub { + format!( + " It is a tool used for the following app: {}", + item.get_integration_type() + .unwrap_or("No integration type".to_string()) + ) + } else { + "".to_string() + } + ); + + let schema = item.get_schema(); + let schema_obj = + backend.transform_schema_for_resources(&schema, resources_cache, resources_types); + + let input_schema_map = match serde_json::to_value(schema_obj) { + Ok(serde_json::Value::Object(map)) => map, + Ok(_) => { + tracing::warn!( + "Schema object for tool '{}' did not serialize to a JSON object, using empty schema.", + path + ); + serde_json::Map::new() + } + Err(e) => { + tracing::error!( + "Failed to serialize schema object for tool '{}': {}. Using empty schema.", + path, + e + ); + serde_json::Map::new() + } + }; + + Tool { + name: Cow::Owned(path), + description: Some(Cow::Owned(description)), + input_schema: Arc::new(input_schema_map), + title: Some(item.get_summary().to_string()), + output_schema: None, + icons: None, + annotations: Some(ToolAnnotations { + title: Some(item.get_summary().to_string()), + read_only_hint: Some(false), // Can modify environment + destructive_hint: Some(true), // Can potentially be destructive + idempotent_hint: Some(false), // Are not guaranteed to be idempotent + open_world_hint: Some(true), // Can interact with external services + }), + meta: None, + } +} From 86ebf9e25a03db99453269832bce030438c677c3 Mon Sep 17 00:00:00 2001 From: Alexander Petric Date: Fri, 16 Jan 2026 16:43:40 -0500 Subject: [PATCH 03/19] fix: transparency issue of instance setting save button (#7594) --- frontend/src/lib/components/SuperadminSettingsInner.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/lib/components/SuperadminSettingsInner.svelte b/frontend/src/lib/components/SuperadminSettingsInner.svelte index 78a47a5675..4a50dbae91 100644 --- a/frontend/src/lib/components/SuperadminSettingsInner.svelte +++ b/frontend/src/lib/components/SuperadminSettingsInner.svelte @@ -380,7 +380,7 @@ {#if tab != 'users'} -
+
+
+
+ {#if !$enterpriseLicense} + + Workspace success handler is a Windmill Enterprise Edition feature that allows you to + run a script whenever any job in the workspace completes successfully. + + {/if} +
+
+ { + successHandlerScriptPath = ev?.detail?.path + }} + /> + +
+
+ +
+ + {#if successHandlerScriptPath} + + {/if} +
+
+
+ {#if !$enterpriseLicense} + + Workspace critical alerts is a Windmill Enterprise Edition feature that sends + notifications to workspace admins when critical events occur. + + {/if} Date: Sat, 17 Jan 2026 21:04:18 +0000 Subject: [PATCH 09/19] fix: handle missing storage key in S3Object for write_s3_file Use .get("storage") instead of direct key access to prevent KeyError when s3object doesn't contain the optional storage field. Co-Authored-By: Claude Opus 4.5 --- python-client/wmill/wmill/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-client/wmill/wmill/client.py b/python-client/wmill/wmill/client.py index cb1b502223..19fee97a41 100644 --- a/python-client/wmill/wmill/client.py +++ b/python-client/wmill/wmill/client.py @@ -975,7 +975,7 @@ class Windmill: ).json() except Exception as e: raise Exception("Could not write file to S3") from e - return S3Object(s3=response["file_key"], storage=s3object["storage"]) + return S3Object(s3=response["file_key"], storage=s3object.get("storage") if s3object else None) def sign_s3_objects(self, s3_objects: list[S3Object | str]) -> list[S3Object]: """Sign S3 objects for use by anonymous users in public apps. From 437025a0786f96ba1105150384827b55111e4181 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 17 Jan 2026 21:08:41 +0000 Subject: [PATCH 10/19] chore(main): release 1.610.0 (#7596) * chore(main): release 1.610.0 * Apply automatic changes --------- Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com> --- CHANGELOG.md | 14 +++++ backend/Cargo.lock | 58 +++++++++---------- backend/Cargo.toml | 4 +- backend/windmill-api/openapi.yaml | 2 +- benchmarks/lib.ts | 2 +- cli/src/main.ts | 2 +- frontend/package-lock.json | 4 +- frontend/package.json | 2 +- lsp/Pipfile | 4 +- openflow.openapi.yaml | 2 +- .../WindmillClient/WindmillClient.psd1 | 2 +- python-client/wmill/pyproject.toml | 2 +- python-client/wmill_pg/pyproject.toml | 2 +- typescript-client/jsr.json | 2 +- typescript-client/package.json | 2 +- version.txt | 2 +- 16 files changed, 60 insertions(+), 46 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ea74046f88..e2dbcfd2d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,19 @@ # Changelog +## [1.610.0](https://github.com/windmill-labs/windmill/compare/v1.609.0...v1.610.0) (2026-01-17) + + +### Features + +* add private npm registry proxy support for ATA in webide ([#7597](https://github.com/windmill-labs/windmill/issues/7597)) ([b3cb41e](https://github.com/windmill-labs/windmill/commit/b3cb41efa4520fd3243a6dbab7c985923dd538e3)) +* add workspace success handler with 60s TTL caching ([#7598](https://github.com/windmill-labs/windmill/issues/7598)) ([73c4ce3](https://github.com/windmill-labs/windmill/commit/73c4ce30127af587510b16fbc7fa223436f6845c)) + + +### Bug Fixes + +* handle missing storage key in S3Object for write_s3_file ([03daa34](https://github.com/windmill-labs/windmill/commit/03daa341eb91118c27e92b1b51731400a0dce30c)) +* improve job deletion performance and batching ([8dd5e81](https://github.com/windmill-labs/windmill/commit/8dd5e81a32b97814eab3b72964807bdfe0ea8b49)) + ## [1.609.0](https://github.com/windmill-labs/windmill/compare/v1.608.0...v1.609.0) (2026-01-16) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 269166b066..aeaf5ec398 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -15243,7 +15243,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill" -version = "1.609.0" +version = "1.610.0" dependencies = [ "anyhow", "aws-sdk-config", @@ -15306,7 +15306,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.609.0" +version = "1.610.0" dependencies = [ "anyhow", "argon2", @@ -15431,7 +15431,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.609.0" +version = "1.610.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -15441,7 +15441,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.609.0" +version = "1.610.0" dependencies = [ "chrono", "lazy_static", @@ -15455,7 +15455,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.609.0" +version = "1.610.0" dependencies = [ "anyhow", "axum 0.7.9", @@ -15474,7 +15474,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.609.0" +version = "1.610.0" dependencies = [ "anyhow", "async-recursion", @@ -15568,7 +15568,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.609.0" +version = "1.610.0" dependencies = [ "regex", "serde", @@ -15583,7 +15583,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.609.0" +version = "1.610.0" dependencies = [ "anyhow", "astral-tokio-tar", @@ -15607,7 +15607,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.609.0" +version = "1.610.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -15623,7 +15623,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.609.0" +version = "1.610.0" dependencies = [ "anyhow", "async-trait", @@ -15643,7 +15643,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.609.0" +version = "1.610.0" dependencies = [ "anyhow", "async-oauth2", @@ -15667,7 +15667,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.609.0" +version = "1.610.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -15676,7 +15676,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.609.0" +version = "1.610.0" dependencies = [ "anyhow", "lazy_static", @@ -15688,7 +15688,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.609.0" +version = "1.610.0" dependencies = [ "anyhow", "serde_json", @@ -15700,7 +15700,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.609.0" +version = "1.610.0" dependencies = [ "anyhow", "gosyn", @@ -15712,7 +15712,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.609.0" +version = "1.610.0" dependencies = [ "anyhow", "lazy_static", @@ -15724,7 +15724,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.609.0" +version = "1.610.0" dependencies = [ "anyhow", "serde_json", @@ -15736,7 +15736,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.609.0" +version = "1.610.0" dependencies = [ "anyhow", "nu-parser", @@ -15747,7 +15747,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.609.0" +version = "1.610.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15758,7 +15758,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.609.0" +version = "1.610.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15771,7 +15771,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.609.0" +version = "1.610.0" dependencies = [ "anyhow", "async-recursion", @@ -15795,7 +15795,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.609.0" +version = "1.610.0" dependencies = [ "anyhow", "lazy_static", @@ -15809,7 +15809,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.609.0" +version = "1.610.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -15826,7 +15826,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.609.0" +version = "1.610.0" dependencies = [ "anyhow", "lazy_static", @@ -15840,7 +15840,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.609.0" +version = "1.610.0" dependencies = [ "anyhow", "lazy_static", @@ -15859,7 +15859,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.609.0" +version = "1.610.0" dependencies = [ "anyhow", "serde", @@ -15870,7 +15870,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.609.0" +version = "1.610.0" dependencies = [ "anyhow", "async-recursion", @@ -15907,7 +15907,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.609.0" +version = "1.610.0" dependencies = [ "wasm-bindgen", "wasm-bindgen-test", @@ -15917,7 +15917,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.609.0" +version = "1.610.0" dependencies = [ "anyhow", "async-once-cell", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index ea178bb1b8..f6d964e8bf 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.609.0" +version = "1.610.0" authors.workspace = true edition.workspace = true @@ -35,7 +35,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal"] [workspace.package] -version = "1.609.0" +version = "1.610.0" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index ed8e8ebde7..1c55f5fb69 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.609.0 + version: 1.610.0 title: Windmill API contact: diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 2da1723d9a..6ee18e925e 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.609.0"; +export const VERSION = "v1.610.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/src/main.ts b/cli/src/main.ts index 023f6439fe..5b93c6699a 100644 --- a/cli/src/main.ts +++ b/cli/src/main.ts @@ -70,7 +70,7 @@ export { // } // }); -export const VERSION = "1.609.0"; +export const VERSION = "1.610.0"; // Re-exported from constants.ts to maintain backwards compatibility export { WM_FORK_PREFIX } from "./core/constants.ts"; diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 7c176ea84c..f1e259717f 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.609.0", + "version": "1.610.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.609.0", + "version": "1.610.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { diff --git a/frontend/package.json b/frontend/package.json index 317db54c46..daaf2d9c53 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.609.0", + "version": "1.610.0", "scripts": { "dev": "vite dev", "build": "vite build", diff --git a/lsp/Pipfile b/lsp/Pipfile index e9f5a06d1c..d6b7cc5749 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,8 +4,8 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.609.0" -wmill_pg = ">=1.609.0" +wmill = ">=1.610.0" +wmill_pg = ">=1.610.0" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index 2cd2cd6713..440c908813 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: '3.0.3' info: - version: 1.609.0 + version: 1.610.0 title: OpenFlow Spec contact: name: Ruben Fiszel diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index b77b9f1c53..f37737fe4d 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. - ModuleVersion = '1.609.0' + ModuleVersion = '1.610.0' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index 4d42c483a1..235135df1d 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.609.0" +version = "1.610.0" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/python-client/wmill_pg/pyproject.toml b/python-client/wmill_pg/pyproject.toml index 9e4564935f..c419d2ef3f 100644 --- a/python-client/wmill_pg/pyproject.toml +++ b/python-client/wmill_pg/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill-pg" -version = "1.609.0" +version = "1.610.0" description = "An extension client for the wmill client library focused on pg" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index fc03e7a8dd..616b0f3b14 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.609.0", + "version": "1.610.0", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./sqlUtils.ts", "./client.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index 60556c9931..3de8c07f08 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.609.0", + "version": "1.610.0", "author": "Ruben Fiszel", "license": "Apache 2.0", "devDependencies": { diff --git a/version.txt b/version.txt index 48096e3546..2473821536 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.609.0 +1.610.0 From 2eac74cef4aa5a987fb16110388f99e912951db8 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 17 Jan 2026 21:18:51 +0000 Subject: [PATCH 11/19] fix: resolve BlobPart type incompatibility between Deno and Node.js Use .slice() on Uint8Array values before passing to File/Blob constructors to create fresh ArrayBuffer-backed arrays, avoiding type errors from ArrayBufferLike vs ArrayBuffer differences in TypeScript definitions. Co-Authored-By: Claude Opus 4.5 --- cli/src/commands/script/script.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cli/src/commands/script/script.ts b/cli/src/commands/script/script.ts index 0bca28c294..820bada9b3 100644 --- a/cli/src/commands/script/script.ts +++ b/cli/src/commands/script/script.ts @@ -314,7 +314,8 @@ export async function handleFile( continue; } log.info(`Adding file: ${file.path.substring(1)}`); - const fil = new File([file.contents as BlobPart], file.path.substring(1)); + // .slice() creates new ArrayBuffer, needed for cross-platform compatibility + const fil = new File([file.contents.slice()], file.path.substring(1)); tarball.append(fil); } const endTime = performance.now(); @@ -541,8 +542,8 @@ async function streamToBlob(stream: ReadableStream): Promise { chunks.push(value); } - // Create a Blob from the chunks - const blob = new Blob(chunks as BlobPart[]); + // Create a Blob from the chunks (.map with slice for cross-platform compatibility) + const blob = new Blob(chunks.map(c => c.slice())); return blob; } From ff771546380ef26dfa443f9d459853048bc8029c Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 17 Jan 2026 21:22:05 +0000 Subject: [PATCH 12/19] fix: use type cast instead of slice() for BlobPart compatibility Avoids unnecessary data copying by using `as any` cast instead of .slice() to work around TypeScript's ArrayBufferLike vs ArrayBuffer type incompatibility between Deno and Node.js. Co-Authored-By: Claude Opus 4.5 --- cli/src/commands/script/script.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cli/src/commands/script/script.ts b/cli/src/commands/script/script.ts index 820bada9b3..0fbf82c84f 100644 --- a/cli/src/commands/script/script.ts +++ b/cli/src/commands/script/script.ts @@ -314,8 +314,8 @@ export async function handleFile( continue; } log.info(`Adding file: ${file.path.substring(1)}`); - // .slice() creates new ArrayBuffer, needed for cross-platform compatibility - const fil = new File([file.contents.slice()], file.path.substring(1)); + // deno-lint-ignore no-explicit-any + const fil = new File([file.contents as any], file.path.substring(1)); tarball.append(fil); } const endTime = performance.now(); @@ -542,8 +542,8 @@ async function streamToBlob(stream: ReadableStream): Promise { chunks.push(value); } - // Create a Blob from the chunks (.map with slice for cross-platform compatibility) - const blob = new Blob(chunks.map(c => c.slice())); + // deno-lint-ignore no-explicit-any + const blob = new Blob(chunks as any); return blob; } From 4d8721c1636c27bb29d00db4bdf263247b451ca2 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 17 Jan 2026 21:30:19 +0000 Subject: [PATCH 13/19] chore(main): release 1.610.1 (#7601) * chore(main): release 1.610.1 * Apply automatic changes --------- Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com> --- CHANGELOG.md | 8 +++ backend/Cargo.lock | 58 +++++++++---------- backend/Cargo.toml | 4 +- backend/windmill-api/openapi.yaml | 2 +- benchmarks/lib.ts | 2 +- cli/src/main.ts | 2 +- frontend/package-lock.json | 4 +- frontend/package.json | 2 +- lsp/Pipfile | 4 +- openflow.openapi.yaml | 2 +- .../WindmillClient/WindmillClient.psd1 | 2 +- python-client/wmill/pyproject.toml | 2 +- python-client/wmill_pg/pyproject.toml | 2 +- typescript-client/jsr.json | 2 +- typescript-client/package.json | 2 +- version.txt | 2 +- 16 files changed, 54 insertions(+), 46 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e2dbcfd2d7..6ca2d357e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [1.610.1](https://github.com/windmill-labs/windmill/compare/v1.610.0...v1.610.1) (2026-01-17) + + +### Bug Fixes + +* resolve BlobPart type incompatibility between Deno and Node.js ([2eac74c](https://github.com/windmill-labs/windmill/commit/2eac74cef4aa5a987fb16110388f99e912951db8)) +* use type cast instead of slice() for BlobPart compatibility ([ff77154](https://github.com/windmill-labs/windmill/commit/ff771546380ef26dfa443f9d459853048bc8029c)) + ## [1.610.0](https://github.com/windmill-labs/windmill/compare/v1.609.0...v1.610.0) (2026-01-17) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index aeaf5ec398..78d498d2b3 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -15243,7 +15243,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill" -version = "1.610.0" +version = "1.610.1" dependencies = [ "anyhow", "aws-sdk-config", @@ -15306,7 +15306,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.610.0" +version = "1.610.1" dependencies = [ "anyhow", "argon2", @@ -15431,7 +15431,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.610.0" +version = "1.610.1" dependencies = [ "reqwest 0.12.28", "serde", @@ -15441,7 +15441,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.610.0" +version = "1.610.1" dependencies = [ "chrono", "lazy_static", @@ -15455,7 +15455,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.610.0" +version = "1.610.1" dependencies = [ "anyhow", "axum 0.7.9", @@ -15474,7 +15474,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.610.0" +version = "1.610.1" dependencies = [ "anyhow", "async-recursion", @@ -15568,7 +15568,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.610.0" +version = "1.610.1" dependencies = [ "regex", "serde", @@ -15583,7 +15583,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.610.0" +version = "1.610.1" dependencies = [ "anyhow", "astral-tokio-tar", @@ -15607,7 +15607,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.610.0" +version = "1.610.1" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -15623,7 +15623,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.610.0" +version = "1.610.1" dependencies = [ "anyhow", "async-trait", @@ -15643,7 +15643,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.610.0" +version = "1.610.1" dependencies = [ "anyhow", "async-oauth2", @@ -15667,7 +15667,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.610.0" +version = "1.610.1" dependencies = [ "convert_case 0.6.0", "serde", @@ -15676,7 +15676,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.610.0" +version = "1.610.1" dependencies = [ "anyhow", "lazy_static", @@ -15688,7 +15688,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.610.0" +version = "1.610.1" dependencies = [ "anyhow", "serde_json", @@ -15700,7 +15700,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.610.0" +version = "1.610.1" dependencies = [ "anyhow", "gosyn", @@ -15712,7 +15712,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.610.0" +version = "1.610.1" dependencies = [ "anyhow", "lazy_static", @@ -15724,7 +15724,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.610.0" +version = "1.610.1" dependencies = [ "anyhow", "serde_json", @@ -15736,7 +15736,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.610.0" +version = "1.610.1" dependencies = [ "anyhow", "nu-parser", @@ -15747,7 +15747,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.610.0" +version = "1.610.1" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15758,7 +15758,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.610.0" +version = "1.610.1" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15771,7 +15771,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.610.0" +version = "1.610.1" dependencies = [ "anyhow", "async-recursion", @@ -15795,7 +15795,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.610.0" +version = "1.610.1" dependencies = [ "anyhow", "lazy_static", @@ -15809,7 +15809,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.610.0" +version = "1.610.1" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -15826,7 +15826,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.610.0" +version = "1.610.1" dependencies = [ "anyhow", "lazy_static", @@ -15840,7 +15840,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.610.0" +version = "1.610.1" dependencies = [ "anyhow", "lazy_static", @@ -15859,7 +15859,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.610.0" +version = "1.610.1" dependencies = [ "anyhow", "serde", @@ -15870,7 +15870,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.610.0" +version = "1.610.1" dependencies = [ "anyhow", "async-recursion", @@ -15907,7 +15907,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.610.0" +version = "1.610.1" dependencies = [ "wasm-bindgen", "wasm-bindgen-test", @@ -15917,7 +15917,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.610.0" +version = "1.610.1" dependencies = [ "anyhow", "async-once-cell", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index f6d964e8bf..28126fe054 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.610.0" +version = "1.610.1" authors.workspace = true edition.workspace = true @@ -35,7 +35,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal"] [workspace.package] -version = "1.610.0" +version = "1.610.1" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 1c55f5fb69..b334483309 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.610.0 + version: 1.610.1 title: Windmill API contact: diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 6ee18e925e..d391e243d4 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.610.0"; +export const VERSION = "v1.610.1"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/src/main.ts b/cli/src/main.ts index 5b93c6699a..867d1fa7fb 100644 --- a/cli/src/main.ts +++ b/cli/src/main.ts @@ -70,7 +70,7 @@ export { // } // }); -export const VERSION = "1.610.0"; +export const VERSION = "1.610.1"; // Re-exported from constants.ts to maintain backwards compatibility export { WM_FORK_PREFIX } from "./core/constants.ts"; diff --git a/frontend/package-lock.json b/frontend/package-lock.json index f1e259717f..8a8ad6f15e 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.610.0", + "version": "1.610.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.610.0", + "version": "1.610.1", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { diff --git a/frontend/package.json b/frontend/package.json index daaf2d9c53..3df060d6c3 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.610.0", + "version": "1.610.1", "scripts": { "dev": "vite dev", "build": "vite build", diff --git a/lsp/Pipfile b/lsp/Pipfile index d6b7cc5749..bca80202ad 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,8 +4,8 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.610.0" -wmill_pg = ">=1.610.0" +wmill = ">=1.610.1" +wmill_pg = ">=1.610.1" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index 440c908813..24e34c8f3a 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: '3.0.3' info: - version: 1.610.0 + version: 1.610.1 title: OpenFlow Spec contact: name: Ruben Fiszel diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index f37737fe4d..57eae4a29b 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. - ModuleVersion = '1.610.0' + ModuleVersion = '1.610.1' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index 235135df1d..09b0d2beec 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.610.0" +version = "1.610.1" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/python-client/wmill_pg/pyproject.toml b/python-client/wmill_pg/pyproject.toml index c419d2ef3f..c4d6b47326 100644 --- a/python-client/wmill_pg/pyproject.toml +++ b/python-client/wmill_pg/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill-pg" -version = "1.610.0" +version = "1.610.1" description = "An extension client for the wmill client library focused on pg" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index 616b0f3b14..3c9618a769 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.610.0", + "version": "1.610.1", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./sqlUtils.ts", "./client.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index 3de8c07f08..2c65914f87 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.610.0", + "version": "1.610.1", "author": "Ruben Fiszel", "license": "Apache 2.0", "devDependencies": { diff --git a/version.txt b/version.txt index 2473821536..3173f5e2c6 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.610.0 +1.610.1 From 1b9d1c56c7e49042677326eb397e10d34a3ddcdf Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sun, 18 Jan 2026 23:08:29 +0000 Subject: [PATCH 14/19] feat: add HashiCorp Vault secret storage integration (#7599) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add HashiCorp Vault secret storage integration - Create SecretBackend trait abstraction for secret storage - Add VaultBackend implementation with CRUD operations - Integrate secret backend into variable CRUD operations - Add migration functions (DB → Vault and Vault → DB) - Add frontend configuration UI for secret backend - Add test connection and migration endpoints --- ...9d39a5a47ca1b6cf3dbfd6988aa0694d7364c.json | 23 + ...a012db51b9e5a255ec0da9a8a0999d668d336.json | 20 + ...84f85ca82636f274f5dce47da5a8320a60088.json | 26 + ...bf24a94eb8a52ceb82cc29dade173ad87569d.json | 23 + ...54b63f8c06c9143b16ce399170c1b5a6b911e.json | 29 + ...aa6f80ecf1628bd2c073d7a237dea9b8e0c65.json | 23 + ...27caa754424e062427edf5ddcb95f9d96888e.json | 29 + ...87f63348a92908e2ca52f5a092149191e6200.json | 16 + ...153c43903f929ae5d62fbba12610f89c36d55.json | 2 +- ...8fe1541bd3f22634af341849303de84c0034d.json | 32 + ...5daa1653e14b6d12013f6e3bb8bf042447a6a.json | 12 + ...3cd956e4c807c3a0031913dbfded80f0e5881.json | 12 + ...fba6ed3b1d573c298f9d7fe8056ba7e32ed81.json | 32 + ...9aae723925270d9d5124cc4cb2c03db61dea8.json | 20 + ...8546bf30074bad903959f7df030e0ac70a86f.json | 32 + ...de229d507784fbcdf46c309907df123e35018.json | 32 + ...fc3113e37dd8009425e558b5f5d1e1543b513.json | 16 + backend/ee-repo-ref.txt | 2 +- backend/tests/fixtures/secret_backend.sql | 31 + backend/tests/secret_backend_integration.rs | 557 ++++++++++++++++++ backend/tests/secret_backend_migration.rs | 323 ++++++++++ backend/windmill-api/Cargo.toml | 2 +- backend/windmill-api/openapi.yaml | 154 +++++ backend/windmill-api/src/lib.rs | 3 + backend/windmill-api/src/resources.rs | 31 +- .../windmill-api/src/secret_backend_ext.rs | 408 +++++++++++++ backend/windmill-api/src/settings.rs | 109 ++++ backend/windmill-api/src/users.rs | 40 ++ backend/windmill-api/src/variables.rs | 97 ++- .../windmill-common/src/global_settings.rs | 1 + backend/windmill-common/src/lib.rs | 1 + .../src/secret_backend/database.rs | 126 ++++ .../windmill-common/src/secret_backend/mod.rs | 133 +++++ .../src/secret_backend/tests.rs | 143 +++++ .../src/secret_backend/vault_oss.rs | 107 ++++ frontend/package-lock.json | 46 +- .../src/lib/components/InstanceSetting.svelte | 3 + .../src/lib/components/instanceSettings.ts | 12 + .../SecretBackendConfig.svelte | 539 +++++++++++++++++ 39 files changed, 3177 insertions(+), 70 deletions(-) create mode 100644 backend/.sqlx/query-020c031c3de6c85577e30421ada9d39a5a47ca1b6cf3dbfd6988aa0694d7364c.json create mode 100644 backend/.sqlx/query-052d42b46d5faba6b41f1fdcbf6a012db51b9e5a255ec0da9a8a0999d668d336.json create mode 100644 backend/.sqlx/query-0600f2a9179f83502c6b13e8e4284f85ca82636f274f5dce47da5a8320a60088.json create mode 100644 backend/.sqlx/query-0c8a3eb810c96230ba3a5466c55bf24a94eb8a52ceb82cc29dade173ad87569d.json create mode 100644 backend/.sqlx/query-146f0e42ada3068a5cdae0ffdbb54b63f8c06c9143b16ce399170c1b5a6b911e.json create mode 100644 backend/.sqlx/query-18aad20ed9cb2dde46f9d899dc4aa6f80ecf1628bd2c073d7a237dea9b8e0c65.json create mode 100644 backend/.sqlx/query-2fcddda99dd0aacf5007ed459cb27caa754424e062427edf5ddcb95f9d96888e.json create mode 100644 backend/.sqlx/query-37e0c601a463819748078b92d1c87f63348a92908e2ca52f5a092149191e6200.json create mode 100644 backend/.sqlx/query-6dd7e38c902f6b8d397aa3e9e698fe1541bd3f22634af341849303de84c0034d.json create mode 100644 backend/.sqlx/query-8f5722afde37d22c56851da23895daa1653e14b6d12013f6e3bb8bf042447a6a.json create mode 100644 backend/.sqlx/query-93ef241a4f624cb76d680418d313cd956e4c807c3a0031913dbfded80f0e5881.json create mode 100644 backend/.sqlx/query-ade0696ec69bec2258d2e3ff86bfba6ed3b1d573c298f9d7fe8056ba7e32ed81.json create mode 100644 backend/.sqlx/query-b030efec7c7f770bf2ed5331a469aae723925270d9d5124cc4cb2c03db61dea8.json create mode 100644 backend/.sqlx/query-b107d602c60c08f0edcaa99695f8546bf30074bad903959f7df030e0ac70a86f.json create mode 100644 backend/.sqlx/query-b10d2ab53bd8d24b0bbcede8211de229d507784fbcdf46c309907df123e35018.json create mode 100644 backend/.sqlx/query-bb289fd24f443f0f8917ec55bc9fc3113e37dd8009425e558b5f5d1e1543b513.json create mode 100644 backend/tests/fixtures/secret_backend.sql create mode 100644 backend/tests/secret_backend_integration.rs create mode 100644 backend/tests/secret_backend_migration.rs create mode 100644 backend/windmill-api/src/secret_backend_ext.rs create mode 100644 backend/windmill-common/src/secret_backend/database.rs create mode 100644 backend/windmill-common/src/secret_backend/mod.rs create mode 100644 backend/windmill-common/src/secret_backend/tests.rs create mode 100644 backend/windmill-common/src/secret_backend/vault_oss.rs create mode 100644 frontend/src/lib/components/instanceSettings/SecretBackendConfig.svelte diff --git a/backend/.sqlx/query-020c031c3de6c85577e30421ada9d39a5a47ca1b6cf3dbfd6988aa0694d7364c.json b/backend/.sqlx/query-020c031c3de6c85577e30421ada9d39a5a47ca1b6cf3dbfd6988aa0694d7364c.json new file mode 100644 index 0000000000..e8edd87bda --- /dev/null +++ b/backend/.sqlx/query-020c031c3de6c85577e30421ada9d39a5a47ca1b6cf3dbfd6988aa0694d7364c.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT value FROM variable WHERE path = $1 AND workspace_id = $2 AND is_secret = true", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "value", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "020c031c3de6c85577e30421ada9d39a5a47ca1b6cf3dbfd6988aa0694d7364c" +} diff --git a/backend/.sqlx/query-052d42b46d5faba6b41f1fdcbf6a012db51b9e5a255ec0da9a8a0999d668d336.json b/backend/.sqlx/query-052d42b46d5faba6b41f1fdcbf6a012db51b9e5a255ec0da9a8a0999d668d336.json new file mode 100644 index 0000000000..7ddd14bb5c --- /dev/null +++ b/backend/.sqlx/query-052d42b46d5faba6b41f1fdcbf6a012db51b9e5a255ec0da9a8a0999d668d336.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*) as count FROM variable WHERE is_secret = true AND value != 'CLEARED'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "052d42b46d5faba6b41f1fdcbf6a012db51b9e5a255ec0da9a8a0999d668d336" +} diff --git a/backend/.sqlx/query-0600f2a9179f83502c6b13e8e4284f85ca82636f274f5dce47da5a8320a60088.json b/backend/.sqlx/query-0600f2a9179f83502c6b13e8e4284f85ca82636f274f5dce47da5a8320a60088.json new file mode 100644 index 0000000000..e5aec65b91 --- /dev/null +++ b/backend/.sqlx/query-0600f2a9179f83502c6b13e8e4284f85ca82636f274f5dce47da5a8320a60088.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT workspace_id, path FROM variable WHERE is_secret = true", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false + ] + }, + "hash": "0600f2a9179f83502c6b13e8e4284f85ca82636f274f5dce47da5a8320a60088" +} diff --git a/backend/.sqlx/query-0c8a3eb810c96230ba3a5466c55bf24a94eb8a52ceb82cc29dade173ad87569d.json b/backend/.sqlx/query-0c8a3eb810c96230ba3a5466c55bf24a94eb8a52ceb82cc29dade173ad87569d.json new file mode 100644 index 0000000000..e68f16713a --- /dev/null +++ b/backend/.sqlx/query-0c8a3eb810c96230ba3a5466c55bf24a94eb8a52ceb82cc29dade173ad87569d.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path FROM variable WHERE path = ANY($1) AND workspace_id = $2 AND is_secret = true", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "TextArray", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "0c8a3eb810c96230ba3a5466c55bf24a94eb8a52ceb82cc29dade173ad87569d" +} diff --git a/backend/.sqlx/query-146f0e42ada3068a5cdae0ffdbb54b63f8c06c9143b16ce399170c1b5a6b911e.json b/backend/.sqlx/query-146f0e42ada3068a5cdae0ffdbb54b63f8c06c9143b16ce399170c1b5a6b911e.json new file mode 100644 index 0000000000..9fd6daded5 --- /dev/null +++ b/backend/.sqlx/query-146f0e42ada3068a5cdae0ffdbb54b63f8c06c9143b16ce399170c1b5a6b911e.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT value, is_secret FROM variable WHERE path = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "value", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "is_secret", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "146f0e42ada3068a5cdae0ffdbb54b63f8c06c9143b16ce399170c1b5a6b911e" +} diff --git a/backend/.sqlx/query-18aad20ed9cb2dde46f9d899dc4aa6f80ecf1628bd2c073d7a237dea9b8e0c65.json b/backend/.sqlx/query-18aad20ed9cb2dde46f9d899dc4aa6f80ecf1628bd2c073d7a237dea9b8e0c65.json new file mode 100644 index 0000000000..d0c8c1fe6b --- /dev/null +++ b/backend/.sqlx/query-18aad20ed9cb2dde46f9d899dc4aa6f80ecf1628bd2c073d7a237dea9b8e0c65.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT is_secret FROM variable WHERE path = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "is_secret", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "18aad20ed9cb2dde46f9d899dc4aa6f80ecf1628bd2c073d7a237dea9b8e0c65" +} diff --git a/backend/.sqlx/query-2fcddda99dd0aacf5007ed459cb27caa754424e062427edf5ddcb95f9d96888e.json b/backend/.sqlx/query-2fcddda99dd0aacf5007ed459cb27caa754424e062427edf5ddcb95f9d96888e.json new file mode 100644 index 0000000000..5a1fbca834 --- /dev/null +++ b/backend/.sqlx/query-2fcddda99dd0aacf5007ed459cb27caa754424e062427edf5ddcb95f9d96888e.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path, value FROM variable\n WHERE path LIKE ('u/' || $1 || '/%')\n AND workspace_id = $2\n AND is_secret = true\n AND value LIKE '$vault:%'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "value", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "2fcddda99dd0aacf5007ed459cb27caa754424e062427edf5ddcb95f9d96888e" +} diff --git a/backend/.sqlx/query-37e0c601a463819748078b92d1c87f63348a92908e2ca52f5a092149191e6200.json b/backend/.sqlx/query-37e0c601a463819748078b92d1c87f63348a92908e2ca52f5a092149191e6200.json new file mode 100644 index 0000000000..d1f3f24b82 --- /dev/null +++ b/backend/.sqlx/query-37e0c601a463819748078b92d1c87f63348a92908e2ca52f5a092149191e6200.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE variable SET value = $1 WHERE path = $2 AND workspace_id = $3 AND is_secret = true", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "37e0c601a463819748078b92d1c87f63348a92908e2ca52f5a092149191e6200" +} diff --git a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json index 713ccb9dd3..36ddb8ab9f 100644 --- a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json +++ b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json @@ -15,7 +15,7 @@ ] }, "nullable": [ - null + true ] }, "hash": "5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55" diff --git a/backend/.sqlx/query-6dd7e38c902f6b8d397aa3e9e698fe1541bd3f22634af341849303de84c0034d.json b/backend/.sqlx/query-6dd7e38c902f6b8d397aa3e9e698fe1541bd3f22634af341849303de84c0034d.json new file mode 100644 index 0000000000..0107528cdd --- /dev/null +++ b/backend/.sqlx/query-6dd7e38c902f6b8d397aa3e9e698fe1541bd3f22634af341849303de84c0034d.json @@ -0,0 +1,32 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT workspace_id, path, value FROM variable WHERE is_secret = true", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "value", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false, + false + ] + }, + "hash": "6dd7e38c902f6b8d397aa3e9e698fe1541bd3f22634af341849303de84c0034d" +} diff --git a/backend/.sqlx/query-8f5722afde37d22c56851da23895daa1653e14b6d12013f6e3bb8bf042447a6a.json b/backend/.sqlx/query-8f5722afde37d22c56851da23895daa1653e14b6d12013f6e3bb8bf042447a6a.json new file mode 100644 index 0000000000..a298c5fb8c --- /dev/null +++ b/backend/.sqlx/query-8f5722afde37d22c56851da23895daa1653e14b6d12013f6e3bb8bf042447a6a.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE variable SET value = 'ROUND_TRIP_CLEARED' WHERE is_secret = true", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "8f5722afde37d22c56851da23895daa1653e14b6d12013f6e3bb8bf042447a6a" +} diff --git a/backend/.sqlx/query-93ef241a4f624cb76d680418d313cd956e4c807c3a0031913dbfded80f0e5881.json b/backend/.sqlx/query-93ef241a4f624cb76d680418d313cd956e4c807c3a0031913dbfded80f0e5881.json new file mode 100644 index 0000000000..288f40474f --- /dev/null +++ b/backend/.sqlx/query-93ef241a4f624cb76d680418d313cd956e4c807c3a0031913dbfded80f0e5881.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE variable SET value = 'CLEARED' WHERE is_secret = true", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "93ef241a4f624cb76d680418d313cd956e4c807c3a0031913dbfded80f0e5881" +} diff --git a/backend/.sqlx/query-ade0696ec69bec2258d2e3ff86bfba6ed3b1d573c298f9d7fe8056ba7e32ed81.json b/backend/.sqlx/query-ade0696ec69bec2258d2e3ff86bfba6ed3b1d573c298f9d7fe8056ba7e32ed81.json new file mode 100644 index 0000000000..3fe61f6f8e --- /dev/null +++ b/backend/.sqlx/query-ade0696ec69bec2258d2e3ff86bfba6ed3b1d573c298f9d7fe8056ba7e32ed81.json @@ -0,0 +1,32 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT workspace_id, path, value FROM variable WHERE is_secret = true AND value IS NOT NULL AND value != ''", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "value", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false, + false + ] + }, + "hash": "ade0696ec69bec2258d2e3ff86bfba6ed3b1d573c298f9d7fe8056ba7e32ed81" +} diff --git a/backend/.sqlx/query-b030efec7c7f770bf2ed5331a469aae723925270d9d5124cc4cb2c03db61dea8.json b/backend/.sqlx/query-b030efec7c7f770bf2ed5331a469aae723925270d9d5124cc4cb2c03db61dea8.json new file mode 100644 index 0000000000..8a0de9e06e --- /dev/null +++ b/backend/.sqlx/query-b030efec7c7f770bf2ed5331a469aae723925270d9d5124cc4cb2c03db61dea8.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*) as count FROM variable WHERE is_secret = true AND value = 'CLEARED'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "b030efec7c7f770bf2ed5331a469aae723925270d9d5124cc4cb2c03db61dea8" +} diff --git a/backend/.sqlx/query-b107d602c60c08f0edcaa99695f8546bf30074bad903959f7df030e0ac70a86f.json b/backend/.sqlx/query-b107d602c60c08f0edcaa99695f8546bf30074bad903959f7df030e0ac70a86f.json new file mode 100644 index 0000000000..705e0b4c5b --- /dev/null +++ b/backend/.sqlx/query-b107d602c60c08f0edcaa99695f8546bf30074bad903959f7df030e0ac70a86f.json @@ -0,0 +1,32 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT workspace_id, path, value FROM variable WHERE is_secret = true AND value != 'CLEARED' ORDER BY workspace_id, path", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "value", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false, + false + ] + }, + "hash": "b107d602c60c08f0edcaa99695f8546bf30074bad903959f7df030e0ac70a86f" +} diff --git a/backend/.sqlx/query-b10d2ab53bd8d24b0bbcede8211de229d507784fbcdf46c309907df123e35018.json b/backend/.sqlx/query-b10d2ab53bd8d24b0bbcede8211de229d507784fbcdf46c309907df123e35018.json new file mode 100644 index 0000000000..534eea5676 --- /dev/null +++ b/backend/.sqlx/query-b10d2ab53bd8d24b0bbcede8211de229d507784fbcdf46c309907df123e35018.json @@ -0,0 +1,32 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT workspace_id, path, value FROM variable WHERE is_secret = true ORDER BY workspace_id, path", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "value", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false, + false + ] + }, + "hash": "b10d2ab53bd8d24b0bbcede8211de229d507784fbcdf46c309907df123e35018" +} diff --git a/backend/.sqlx/query-bb289fd24f443f0f8917ec55bc9fc3113e37dd8009425e558b5f5d1e1543b513.json b/backend/.sqlx/query-bb289fd24f443f0f8917ec55bc9fc3113e37dd8009425e558b5f5d1e1543b513.json new file mode 100644 index 0000000000..a687e1a7e3 --- /dev/null +++ b/backend/.sqlx/query-bb289fd24f443f0f8917ec55bc9fc3113e37dd8009425e558b5f5d1e1543b513.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE variable SET value = $1 WHERE path = $2 AND workspace_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "bb289fd24f443f0f8917ec55bc9fc3113e37dd8009425e558b5f5d1e1543b513" +} diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 56e9d8a0be..60e3437748 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -da1518ed54410478ca209f1d06298a1407be38a8 \ No newline at end of file +54a01bfeb32cedb779c8d39ae2fbfd26c8be0482 \ No newline at end of file diff --git a/backend/tests/fixtures/secret_backend.sql b/backend/tests/fixtures/secret_backend.sql new file mode 100644 index 0000000000..0bc5a6f52a --- /dev/null +++ b/backend/tests/fixtures/secret_backend.sql @@ -0,0 +1,31 @@ +-- Fixture for secret backend migration tests +-- Sets up test secrets in the variable table + +-- Create a second workspace for testing workspace isolation +INSERT INTO workspace (id, name, owner) +VALUES ('test-workspace-2', 'test-workspace-2', 'test-user') +ON CONFLICT DO NOTHING; + +INSERT INTO workspace_settings (workspace_id) +VALUES ('test-workspace-2') +ON CONFLICT DO NOTHING; + +INSERT INTO workspace_key(workspace_id, kind, key) +VALUES ('test-workspace-2', 'cloud', 'test-key-2') +ON CONFLICT DO NOTHING; + +-- Insert test secrets for workspace 1 +-- Note: The 'value' column stores encrypted values in production, +-- but for tests we'll use plain text that the migration will handle +INSERT INTO variable (workspace_id, path, value, is_secret, description, extra_perms) +VALUES + ('test-workspace', 'u/test-user/db_password', 'encrypted-db-pass-123', true, 'Database password', '{}'), + ('test-workspace', 'u/test-user/api_key', 'encrypted-api-key-abc', true, 'API key for external service', '{}'), + ('test-workspace', 'u/test-user/public_var', 'not-a-secret', false, 'A non-secret variable', '{}') +ON CONFLICT DO NOTHING; + +-- Insert test secrets for workspace 2 (to test isolation) +INSERT INTO variable (workspace_id, path, value, is_secret, description, extra_perms) +VALUES + ('test-workspace-2', 'u/test-user/other_secret', 'encrypted-other-secret', true, 'Secret in workspace 2', '{}') +ON CONFLICT DO NOTHING; diff --git a/backend/tests/secret_backend_integration.rs b/backend/tests/secret_backend_integration.rs new file mode 100644 index 0000000000..e6deb84ff9 --- /dev/null +++ b/backend/tests/secret_backend_integration.rs @@ -0,0 +1,557 @@ +//! Integration tests for HashiCorp Vault secret backend. +//! +//! These tests require: +//! 1. A PostgreSQL database (handled by sqlx test framework) +//! 2. A running HashiCorp Vault instance +//! 3. The RUN_VAULT_TESTS=1 environment variable to be set +//! +//! Environment variables: +//! - RUN_VAULT_TESTS=1 - Required to run the tests +//! - VAULT_ADDR - Vault server address (default: http://127.0.0.1:8200) +//! - VAULT_TOKEN - Static token for static token tests (default: test-root-token) +//! - BASE_URL - Windmill instance URL for JWT tests (default: http://localhost:8000) +//! +//! Run tests (static token mode): +//! ```bash +//! RUN_VAULT_TESTS=1 VAULT_TOKEN=your-token cargo test -p windmill \ +//! secret_backend_integration --features private,enterprise -- --nocapture +//! ``` +//! +//! Run tests (JWT mode - requires Windmill instance running for JWKS endpoint): +//! ```bash +//! RUN_VAULT_TESTS=1 BASE_URL=http://localhost:8000 cargo test -p windmill \ +//! secret_backend_integration --features private,enterprise,openidconnect -- --nocapture +//! ``` + +#[cfg(all(feature = "private", feature = "enterprise"))] +mod tests { + use sqlx::{Pool, Postgres}; + use std::collections::HashMap; + use windmill_common::secret_backend::{ + migrate_secrets_to_database, migrate_secrets_to_vault, test_vault_connection, + SecretBackend, VaultBackend, VaultSettings, + }; + + /// Check if vault tests should run (requires RUN_VAULT_TESTS=1 env var) + fn should_run_vault_tests() -> bool { + std::env::var("RUN_VAULT_TESTS") + .map(|v| v == "1" || v.to_lowercase() == "true") + .unwrap_or(false) + } + + /// Set up BASE_URL for JWT tests (required for OIDC issuer URL generation) + async fn setup_base_url() { + let base_url = std::env::var("BASE_URL") + .unwrap_or_else(|_| "http://localhost:8000".to_string()); + let mut url = windmill_common::BASE_URL.write().await; + *url = base_url; + } + + /// Skip test if RUN_VAULT_TESTS is not set + macro_rules! skip_if_no_vault { + () => { + if !should_run_vault_tests() { + println!("Skipping test: RUN_VAULT_TESTS=1 not set"); + println!("To run vault tests: RUN_VAULT_TESTS=1 cargo test ..."); + return; + } + }; + } + + fn vault_settings_static_token() -> VaultSettings { + VaultSettings { + address: std::env::var("VAULT_ADDR") + .unwrap_or_else(|_| "http://127.0.0.1:8200".to_string()), + mount_path: "windmill".to_string(), + jwt_role: None, // Static token mode + namespace: None, + token: Some( + std::env::var("VAULT_TOKEN").unwrap_or_else(|_| "test-root-token".to_string()), + ), + } + } + + fn vault_settings_jwt() -> VaultSettings { + VaultSettings { + address: std::env::var("VAULT_ADDR") + .unwrap_or_else(|_| "http://127.0.0.1:8200".to_string()), + mount_path: "windmill".to_string(), + jwt_role: Some("windmill-secrets".to_string()), // JWT mode + namespace: None, + token: None, // No static token - use JWT + } + } + + // ==================== Static Token Tests ==================== + + /// Test Vault connection with static token + #[sqlx::test(fixtures("base", "secret_backend"))] + async fn test_vault_connection_static_token(db: Pool) { + skip_if_no_vault!(); + + let settings = vault_settings_static_token(); + println!("Testing Vault connection with static token..."); + println!(" Address: {}", settings.address); + + let result = test_vault_connection(&settings, Some(&db)).await; + assert!( + result.is_ok(), + "Failed to connect to Vault: {:?}", + result.err() + ); + println!("✓ Successfully connected to Vault with static token"); + } + + /// Test basic CRUD operations with static token + #[sqlx::test(fixtures("base", "secret_backend"))] + async fn test_vault_crud_static_token(_db: Pool) { + skip_if_no_vault!(); + + let settings = vault_settings_static_token(); + let backend = VaultBackend::new(settings); + + let workspace_id = "test-crud-static"; + let path = "test-secret"; + let value = "my-super-secret-value-123"; + + println!("Testing CRUD with static token..."); + + // Create + println!(" Creating secret..."); + backend + .set_secret(workspace_id, path, value) + .await + .expect("Failed to create secret"); + println!(" ✓ Created"); + + // Read + println!(" Reading secret..."); + let read_value = backend + .get_secret(workspace_id, path) + .await + .expect("Failed to read secret"); + assert_eq!(read_value, value); + println!(" ✓ Read (value matches)"); + + // Update + println!(" Updating secret..."); + let new_value = "updated-secret-value-456"; + backend + .set_secret(workspace_id, path, new_value) + .await + .expect("Failed to update secret"); + let updated = backend + .get_secret(workspace_id, path) + .await + .expect("Failed to read updated secret"); + assert_eq!(updated, new_value); + println!(" ✓ Updated"); + + // Delete + println!(" Deleting secret..."); + backend + .delete_secret(workspace_id, path) + .await + .expect("Failed to delete secret"); + let result = backend.get_secret(workspace_id, path).await; + assert!(result.is_err(), "Secret should be deleted"); + println!(" ✓ Deleted"); + + println!("✓ CRUD operations successful with static token"); + } + + // ==================== JWT Auth Tests ==================== + + /// Test Vault connection with JWT authentication + #[sqlx::test(fixtures("base", "secret_backend"))] + async fn test_vault_connection_jwt(db: Pool) { + skip_if_no_vault!(); + setup_base_url().await; + + let settings = vault_settings_jwt(); + println!("Testing Vault connection with JWT auth..."); + println!(" Address: {}", settings.address); + println!(" JWT Role: {:?}", settings.jwt_role); + println!(" BASE_URL: {}", windmill_common::BASE_URL.read().await.clone()); + + let result = test_vault_connection(&settings, Some(&db)).await; + assert!( + result.is_ok(), + "Failed to connect to Vault with JWT: {:?}", + result.err() + ); + println!("✓ Successfully connected to Vault with JWT auth"); + } + + /// Test basic CRUD operations with JWT authentication + #[cfg(feature = "openidconnect")] + #[sqlx::test(fixtures("base", "secret_backend"))] + async fn test_vault_crud_jwt(db: Pool) { + skip_if_no_vault!(); + setup_base_url().await; + + let settings = vault_settings_jwt(); + let backend = VaultBackend::new_with_db(settings, db.clone()); + + let workspace_id = "test-crud-jwt"; + let path = "jwt-test-secret"; + let value = "jwt-authenticated-secret-value"; + + println!("Testing CRUD with JWT auth..."); + + // Create + println!(" Creating secret..."); + backend + .set_secret(workspace_id, path, value) + .await + .expect("Failed to create secret with JWT"); + println!(" ✓ Created"); + + // Read + println!(" Reading secret..."); + let read_value = backend + .get_secret(workspace_id, path) + .await + .expect("Failed to read secret with JWT"); + assert_eq!(read_value, value); + println!(" ✓ Read (value matches)"); + + // Delete (cleanup) + println!(" Deleting secret..."); + backend + .delete_secret(workspace_id, path) + .await + .expect("Failed to delete secret with JWT"); + println!(" ✓ Deleted"); + + println!("✓ CRUD operations successful with JWT auth"); + } + + // ==================== Migration Tests ==================== + + /// Test migration from database to Vault + #[sqlx::test(fixtures("base", "secret_backend"))] + async fn test_migrate_db_to_vault(db: Pool) { + skip_if_no_vault!(); + + let settings = vault_settings_static_token(); + + // Verify Vault connection + test_vault_connection(&settings, Some(&db)) + .await + .expect("Failed to connect to Vault"); + + // Check initial state + let secrets_before = sqlx::query!( + "SELECT workspace_id, path, value FROM variable WHERE is_secret = true ORDER BY workspace_id, path" + ) + .fetch_all(&db) + .await + .expect("Failed to query secrets"); + + println!( + "Found {} secrets in database before migration:", + secrets_before.len() + ); + for s in &secrets_before { + println!(" - {}/{}: {} chars", s.workspace_id, s.path, s.value.len()); + } + + // Run migration + println!("\nMigrating secrets to Vault..."); + let report = migrate_secrets_to_vault(&db, &settings) + .await + .expect("Migration to Vault failed"); + + println!("Migration report:"); + println!(" Total secrets: {}", report.total_secrets); + println!(" Migrated: {}", report.migrated_count); + println!(" Failed: {}", report.failed_count); + + if !report.failures.is_empty() { + println!(" Failures:"); + for f in &report.failures { + println!(" - {}/{}: {}", f.workspace_id, f.path, f.error); + } + } + + assert_eq!(report.failed_count, 0, "Migration had failures"); + assert!(report.migrated_count > 0, "No secrets were migrated"); + + // Verify secrets in Vault + println!("\nVerifying secrets in Vault..."); + let vault_backend = VaultBackend::new(settings.clone()); + + for secret in &secrets_before { + let result = vault_backend + .get_secret(&secret.workspace_id, &secret.path) + .await; + assert!( + result.is_ok(), + "Failed to read secret {}/{} from Vault: {:?}", + secret.workspace_id, + secret.path, + result.err() + ); + println!( + " ✓ {}/{} exists in Vault", + secret.workspace_id, secret.path + ); + } + + println!("\n✓ Migration to Vault completed successfully"); + } + + /// Test migration from Vault back to database + #[sqlx::test(fixtures("base", "secret_backend"))] + async fn test_migrate_vault_to_db(db: Pool) { + skip_if_no_vault!(); + + let settings = vault_settings_static_token(); + + test_vault_connection(&settings, Some(&db)) + .await + .expect("Failed to connect to Vault"); + + // First migrate TO Vault + println!("Setting up: migrating secrets to Vault first..."); + let to_vault = migrate_secrets_to_vault(&db, &settings) + .await + .expect("Initial migration to Vault failed"); + assert!(to_vault.migrated_count > 0, "No secrets to test with"); + println!(" Migrated {} secrets to Vault", to_vault.migrated_count); + + // Clear database values + println!("\nClearing database secret values..."); + sqlx::query!("UPDATE variable SET value = 'CLEARED' WHERE is_secret = true") + .execute(&db) + .await + .expect("Failed to clear values"); + + // Migrate back from Vault + println!("\nMigrating secrets from Vault to database..."); + let report = migrate_secrets_to_database(&db, &settings) + .await + .expect("Migration to database failed"); + + println!("Migration report:"); + println!(" Total secrets: {}", report.total_secrets); + println!(" Migrated: {}", report.migrated_count); + println!(" Failed: {}", report.failed_count); + + assert_eq!(report.failed_count, 0, "Migration had failures"); + assert!(report.migrated_count > 0, "No secrets were migrated"); + + // Verify restored + let restored = sqlx::query!( + "SELECT COUNT(*) as count FROM variable WHERE is_secret = true AND value != 'CLEARED'" + ) + .fetch_one(&db) + .await + .expect("Failed to count restored"); + + assert!( + restored.count.unwrap_or(0) > 0, + "No secrets were restored in database" + ); + + println!("\n✓ Migration to database completed successfully"); + } + + // ==================== Variable Rename Tests ==================== + + /// Test renaming a variable path in Vault + #[sqlx::test(fixtures("base", "secret_backend"))] + async fn test_variable_rename(db: Pool) { + skip_if_no_vault!(); + let _ = &db; // suppress unused warning + + let settings = vault_settings_static_token(); + let backend = VaultBackend::new(settings); + + let workspace_id = "test-workspace"; + let old_path = "u/test-user/old_secret_name"; + let new_path = "u/test-user/new_secret_name"; + let value = "secret-value-for-rename-test"; + + println!("Testing variable rename in Vault..."); + + // Create secret at old path + println!(" Creating secret at old path: {}", old_path); + backend + .set_secret(workspace_id, old_path, value) + .await + .expect("Failed to create secret"); + + // Verify it exists + let read_value = backend + .get_secret(workspace_id, old_path) + .await + .expect("Failed to read secret at old path"); + assert_eq!(read_value, value); + println!(" ✓ Secret exists at old path"); + + // Simulate rename: read from old, write to new, delete old + println!(" Renaming: {} -> {}", old_path, new_path); + let secret_value = backend + .get_secret(workspace_id, old_path) + .await + .expect("Failed to read for rename"); + + backend + .set_secret(workspace_id, new_path, &secret_value) + .await + .expect("Failed to write to new path"); + + backend + .delete_secret(workspace_id, old_path) + .await + .expect("Failed to delete old path"); + + // Verify old path is gone + let old_result = backend.get_secret(workspace_id, old_path).await; + assert!(old_result.is_err(), "Old path should not exist"); + println!(" ✓ Old path deleted"); + + // Verify new path exists with correct value + let new_value = backend + .get_secret(workspace_id, new_path) + .await + .expect("Failed to read new path"); + assert_eq!(new_value, value); + println!(" ✓ New path exists with correct value"); + + // Cleanup + backend + .delete_secret(workspace_id, new_path) + .await + .expect("Failed to cleanup"); + + println!("\n✓ Variable rename completed successfully"); + } + + // ==================== Full Round Trip Test ==================== + + /// Test full round-trip: DB -> Vault -> DB with verification + #[sqlx::test(fixtures("base", "secret_backend"))] + async fn test_full_round_trip(db: Pool) { + skip_if_no_vault!(); + + let settings = vault_settings_static_token(); + + test_vault_connection(&settings, Some(&db)) + .await + .expect("Failed to connect to Vault"); + + // Get original secrets + let original: HashMap<(String, String), String> = sqlx::query!( + "SELECT workspace_id, path, value FROM variable WHERE is_secret = true" + ) + .fetch_all(&db) + .await + .expect("Failed to query") + .into_iter() + .map(|r| ((r.workspace_id, r.path), r.value)) + .collect(); + + println!("Original secrets: {} entries", original.len()); + + // Step 1: DB -> Vault + println!("\n=== Step 1: Migrate DB -> Vault ==="); + let to_vault = migrate_secrets_to_vault(&db, &settings) + .await + .expect("Migration to Vault failed"); + println!("Migrated {} secrets to Vault", to_vault.migrated_count); + assert_eq!(to_vault.failed_count, 0); + + // Step 2: Clear DB + println!("\n=== Step 2: Clear database values ==="); + sqlx::query!("UPDATE variable SET value = 'ROUND_TRIP_CLEARED' WHERE is_secret = true") + .execute(&db) + .await + .expect("Failed to clear"); + + // Step 3: Vault -> DB + println!("\n=== Step 3: Migrate Vault -> DB ==="); + let to_db = migrate_secrets_to_database(&db, &settings) + .await + .expect("Migration to database failed"); + println!("Migrated {} secrets to database", to_db.migrated_count); + assert_eq!(to_db.failed_count, 0); + + // Step 4: Verify + println!("\n=== Step 4: Verify round-trip integrity ==="); + let restored: HashMap<(String, String), String> = sqlx::query!( + "SELECT workspace_id, path, value FROM variable WHERE is_secret = true" + ) + .fetch_all(&db) + .await + .expect("Failed to query") + .into_iter() + .map(|r| ((r.workspace_id, r.path), r.value)) + .collect(); + + for ((ws, path), _) in &original { + let restored_value = restored + .get(&(ws.clone(), path.clone())) + .expect(&format!("Secret {}/{} not found after round-trip", ws, path)); + + assert_ne!( + restored_value, "ROUND_TRIP_CLEARED", + "Secret {}/{} was not restored", + ws, path + ); + println!(" ✓ {}/{}: restored", ws, path); + } + + println!("\n✓ Full round-trip completed successfully!"); + } + + // ==================== Workspace Isolation Test ==================== + + /// Test that workspace isolation is maintained + #[sqlx::test(fixtures("base", "secret_backend"))] + async fn test_workspace_isolation(db: Pool) { + skip_if_no_vault!(); + + let settings = vault_settings_static_token(); + let backend = VaultBackend::new(settings.clone()); + + // First migrate secrets to Vault + migrate_secrets_to_vault(&db, &settings) + .await + .expect("Migration failed"); + + println!("Testing workspace isolation..."); + + // Try to access workspace-2 secret from workspace-1 path (should fail) + let cross_access = backend + .get_secret("test-workspace", "u/test-user/other_secret") + .await; + + assert!( + cross_access.is_err(), + "Cross-workspace access should fail!" + ); + println!("✓ Cross-workspace access correctly denied"); + + // Verify own workspace access works + let ws1 = backend + .get_secret("test-workspace", "u/test-user/db_password") + .await; + assert!(ws1.is_ok(), "Same-workspace access should work"); + println!("✓ Same-workspace access works"); + + println!("\n✓ Workspace isolation verified!"); + } +} + +// OSS version - just a placeholder to avoid compilation errors +#[cfg(not(all(feature = "private", feature = "enterprise")))] +mod tests { + #[test] + fn test_vault_requires_enterprise() { + println!("Vault integration tests require Enterprise Edition features"); + println!("Run with: cargo test --features private,enterprise,openidconnect"); + } +} diff --git a/backend/tests/secret_backend_migration.rs b/backend/tests/secret_backend_migration.rs new file mode 100644 index 0000000000..d78da8603c --- /dev/null +++ b/backend/tests/secret_backend_migration.rs @@ -0,0 +1,323 @@ +//! Integration tests for secret backend migration between database and HashiCorp Vault. +//! +//! These tests require: +//! 1. A PostgreSQL database (handled by sqlx test framework) +//! 2. A running HashiCorp Vault instance at http://127.0.0.1:8200 +//! +//! To run these tests: +//! ```bash +//! # Start Vault in dev mode +//! podman run -d --name vault-test -p 8200:8200 \ +//! -e VAULT_DEV_ROOT_TOKEN_ID=test-root-token \ +//! docker.io/hashicorp/vault:latest +//! +//! # Enable KV v2 secrets engine +//! curl -s -H "X-Vault-Token: test-root-token" -X POST \ +//! --data '{"type":"kv-v2"}' \ +//! http://127.0.0.1:8200/v1/sys/mounts/windmill +//! +//! # Run the tests +//! cargo test -p windmill secret_backend_migration -- --ignored --nocapture +//! ``` + +use sqlx::{Pool, Postgres}; +use windmill_common::error::Result; +use windmill_common::secret_backend::{ + vault_oss::{migrate_secrets_to_database, migrate_secrets_to_vault, test_vault_connection, VaultBackend}, + SecretBackend, VaultSettings, +}; + +mod common; + +fn test_vault_settings() -> VaultSettings { + VaultSettings { + address: std::env::var("VAULT_ADDR").unwrap_or_else(|_| "http://127.0.0.1:8200".to_string()), + mount_path: "windmill".to_string(), + jwt_role: Some("windmill-secrets".to_string()), + namespace: None, + token: Some( + std::env::var("VAULT_TOKEN").unwrap_or_else(|_| "test-root-token".to_string()), + ), + } +} + +/// Test that we can connect to Vault +#[sqlx::test(fixtures("base", "secret_backend"))] +#[ignore = "requires running Vault instance"] +async fn test_vault_connection_works(db: Pool) { + let settings = test_vault_settings(); + + let result = test_vault_connection(&settings, Some(&db)).await; + assert!(result.is_ok(), "Failed to connect to Vault: {:?}", result.err()); + println!("✓ Successfully connected to Vault at {}", settings.address); +} + +/// Test migration from database to Vault +#[sqlx::test(fixtures("base", "secret_backend"))] +#[ignore = "requires running Vault instance"] +async fn test_migrate_db_to_vault(db: Pool) { + let settings = test_vault_settings(); + + // First verify we can connect to Vault + test_vault_connection(&settings, Some(&db)) + .await + .expect("Failed to connect to Vault"); + + // Check initial state - secrets should exist in database + let secrets_before = sqlx::query!( + "SELECT workspace_id, path, value FROM variable WHERE is_secret = true ORDER BY workspace_id, path" + ) + .fetch_all(&db) + .await + .expect("Failed to query secrets"); + + println!("Found {} secrets in database before migration:", secrets_before.len()); + for s in &secrets_before { + println!(" - {}/{}: {} chars", s.workspace_id, s.path, s.value.len()); + } + + // Run migration to Vault + println!("\nMigrating secrets to Vault..."); + let report = migrate_secrets_to_vault(&db, &settings) + .await + .expect("Migration to Vault failed"); + + println!("Migration report:"); + println!(" Total secrets: {}", report.total_secrets); + println!(" Migrated: {}", report.migrated_count); + println!(" Failed: {}", report.failed_count); + + if !report.failures.is_empty() { + println!(" Failures:"); + for f in &report.failures { + println!(" - {}/{}: {}", f.workspace_id, f.path, f.error); + } + } + + assert_eq!(report.failed_count, 0, "Migration had failures"); + assert!(report.migrated_count > 0, "No secrets were migrated"); + + // Verify secrets are in Vault + println!("\nVerifying secrets in Vault..."); + let vault_backend = VaultBackend::new(settings.clone()); + + for secret in &secrets_before { + let result: Result = vault_backend + .get_secret(&secret.workspace_id, &secret.path) + .await; + assert!( + result.is_ok(), + "Failed to read secret {}/{} from Vault: {:?}", + secret.workspace_id, + secret.path, + result.err() + ); + println!(" ✓ {}/{} exists in Vault", secret.workspace_id, secret.path); + } + + println!("\n✓ Migration to Vault completed successfully"); +} + +/// Test migration from Vault to database +#[sqlx::test(fixtures("base", "secret_backend"))] +#[ignore = "requires running Vault instance"] +async fn test_migrate_vault_to_db(db: Pool) { + let settings = test_vault_settings(); + + // First verify we can connect to Vault + test_vault_connection(&settings, Some(&db)) + .await + .expect("Failed to connect to Vault"); + + // First, migrate secrets TO Vault so we have something to migrate back + println!("Setting up: migrating secrets to Vault first..."); + let to_vault_report = migrate_secrets_to_vault(&db, &settings) + .await + .expect("Initial migration to Vault failed"); + assert!(to_vault_report.migrated_count > 0, "No secrets to test with"); + println!(" Migrated {} secrets to Vault", to_vault_report.migrated_count); + + // Clear the database values to simulate fresh migration back + println!("\nClearing database secret values..."); + sqlx::query!("UPDATE variable SET value = 'CLEARED' WHERE is_secret = true") + .execute(&db) + .await + .expect("Failed to clear database values"); + + // Verify they were cleared + let cleared = sqlx::query!( + "SELECT COUNT(*) as count FROM variable WHERE is_secret = true AND value = 'CLEARED'" + ) + .fetch_one(&db) + .await + .expect("Failed to count cleared"); + println!(" Cleared {} secret values in database", cleared.count.unwrap_or(0)); + + // Now migrate from Vault back to database + println!("\nMigrating secrets from Vault to database..."); + let report = migrate_secrets_to_database(&db, &settings) + .await + .expect("Migration to database failed"); + + println!("Migration report:"); + println!(" Total secrets: {}", report.total_secrets); + println!(" Migrated: {}", report.migrated_count); + println!(" Failed: {}", report.failed_count); + + if !report.failures.is_empty() { + println!(" Failures:"); + for f in &report.failures { + println!(" - {}/{}: {}", f.workspace_id, f.path, f.error); + } + } + + assert_eq!(report.failed_count, 0, "Migration had failures"); + assert!(report.migrated_count > 0, "No secrets were migrated"); + + // Verify secrets are restored in database + println!("\nVerifying secrets in database..."); + let secrets_after = sqlx::query!( + "SELECT workspace_id, path, value FROM variable WHERE is_secret = true AND value != 'CLEARED' ORDER BY workspace_id, path" + ) + .fetch_all(&db) + .await + .expect("Failed to query restored secrets"); + + assert!( + !secrets_after.is_empty(), + "No secrets were restored in database" + ); + + for s in &secrets_after { + println!(" ✓ {}/{}: {} chars", s.workspace_id, s.path, s.value.len()); + } + + println!("\n✓ Migration to database completed successfully"); +} + +/// Test full round-trip migration: DB -> Vault -> DB +#[sqlx::test(fixtures("base", "secret_backend"))] +#[ignore = "requires running Vault instance"] +async fn test_full_round_trip_migration(db: Pool) { + let settings = test_vault_settings(); + + // Verify Vault connection + test_vault_connection(&settings, Some(&db)) + .await + .expect("Failed to connect to Vault"); + + // Get original secrets + let original_secrets: std::collections::HashMap<(String, String), String> = sqlx::query!( + "SELECT workspace_id, path, value FROM variable WHERE is_secret = true" + ) + .fetch_all(&db) + .await + .expect("Failed to query original secrets") + .into_iter() + .map(|r| ((r.workspace_id, r.path), r.value)) + .collect(); + + println!("Original secrets: {} entries", original_secrets.len()); + + // Step 1: Migrate to Vault + println!("\n=== Step 1: Migrate DB -> Vault ==="); + let to_vault = migrate_secrets_to_vault(&db, &settings) + .await + .expect("Migration to Vault failed"); + println!("Migrated {} secrets to Vault", to_vault.migrated_count); + assert_eq!(to_vault.failed_count, 0); + + // Step 2: Clear database values + println!("\n=== Step 2: Clear database values ==="); + sqlx::query!("UPDATE variable SET value = 'ROUND_TRIP_CLEARED' WHERE is_secret = true") + .execute(&db) + .await + .expect("Failed to clear values"); + + // Step 3: Migrate back from Vault + println!("\n=== Step 3: Migrate Vault -> DB ==="); + let to_db = migrate_secrets_to_database(&db, &settings) + .await + .expect("Migration to database failed"); + println!("Migrated {} secrets to database", to_db.migrated_count); + assert_eq!(to_db.failed_count, 0); + + // Step 4: Verify round-trip integrity + println!("\n=== Step 4: Verify round-trip integrity ==="); + let restored_secrets: std::collections::HashMap<(String, String), String> = sqlx::query!( + "SELECT workspace_id, path, value FROM variable WHERE is_secret = true" + ) + .fetch_all(&db) + .await + .expect("Failed to query restored secrets") + .into_iter() + .map(|r| ((r.workspace_id, r.path), r.value)) + .collect(); + + // Compare original and restored + for ((ws, path), _original_value) in &original_secrets { + let restored_value = restored_secrets + .get(&(ws.clone(), path.clone())) + .expect(&format!("Secret {}/{} not found after round-trip", ws, path)); + + // Note: Values might differ slightly due to encryption/decryption + // but they should not be the cleared value + assert_ne!( + restored_value, "ROUND_TRIP_CLEARED", + "Secret {}/{} was not restored", + ws, path + ); + println!(" ✓ {}/{}: restored ({} chars)", ws, path, restored_value.len()); + } + + println!("\n✓ Full round-trip migration completed successfully!"); + println!(" Original secrets: {}", original_secrets.len()); + println!(" Restored secrets: {}", restored_secrets.len()); +} + +/// Test that workspace isolation is maintained during migration +#[sqlx::test(fixtures("base", "secret_backend"))] +#[ignore = "requires running Vault instance"] +async fn test_workspace_isolation(db: Pool) { + let settings = test_vault_settings(); + + test_vault_connection(&settings, Some(&db)) + .await + .expect("Failed to connect to Vault"); + + // Migrate all secrets to Vault + let report = migrate_secrets_to_vault(&db, &settings) + .await + .expect("Migration failed"); + + println!("Migrated {} secrets across workspaces", report.migrated_count); + + // Verify workspace isolation in Vault + let vault_backend = VaultBackend::new(settings.clone()); + + // Try to access test-workspace-2 secret from test-workspace path (should fail) + let cross_workspace_result: Result = vault_backend + .get_secret("test-workspace", "u/test-user/other_secret") + .await; + + assert!( + cross_workspace_result.is_err(), + "Cross-workspace access should fail - workspace isolation violated!" + ); + println!("✓ Cross-workspace access correctly denied"); + + // Verify each workspace's secrets are accessible from their own workspace + let ws1_result: Result = vault_backend + .get_secret("test-workspace", "u/test-user/db_password") + .await; + assert!(ws1_result.is_ok(), "test-workspace secret should be accessible"); + println!("✓ test-workspace secrets accessible"); + + let ws2_result: Result = vault_backend + .get_secret("test-workspace-2", "u/test-user/other_secret") + .await; + assert!(ws2_result.is_ok(), "test-workspace-2 secret should be accessible"); + println!("✓ test-workspace-2 secrets accessible"); + + println!("\n✓ Workspace isolation verified!"); +} diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml index 087d904d3f..ae4372ac24 100644 --- a/backend/windmill-api/Cargo.toml +++ b/backend/windmill-api/Cargo.toml @@ -10,7 +10,7 @@ path = "src/lib.rs" [features] default = [] -private = ["windmill-audit/private"] +private = ["windmill-audit/private", "windmill-common/private"] enterprise = ["windmill-queue/enterprise", "windmill-audit/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "windmill-worker/enterprise"] stripe = [] agent_worker_server = [] diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index b334483309..5f635d38a9 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1411,6 +1411,83 @@ paths: items: $ref: "#/components/schemas/GlobalSetting" + /.well-known/jwks.json: + get: + summary: get JWKS for Vault JWT authentication + operationId: getJwks + tags: + - setting + responses: + "200": + description: JSON Web Key Set + content: + application/json: + schema: + $ref: "#/components/schemas/JwksResponse" + + /settings/test_secret_backend: + post: + summary: test secret backend connection (HashiCorp Vault) + operationId: testSecretBackend + tags: + - setting + requestBody: + description: Vault settings to test + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/VaultSettings" + responses: + "200": + description: connection successful + content: + text/plain: + schema: + type: string + + /settings/migrate_secrets_to_vault: + post: + summary: migrate secrets from database to HashiCorp Vault + operationId: migrateSecretsToVault + tags: + - setting + requestBody: + description: Vault settings for migration target + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/VaultSettings" + responses: + "200": + description: migration report + content: + application/json: + schema: + $ref: "#/components/schemas/SecretMigrationReport" + + /settings/migrate_secrets_to_database: + post: + summary: migrate secrets from HashiCorp Vault to database + operationId: migrateSecretsToDatabase + tags: + - setting + requestBody: + description: Vault settings for migration source + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/VaultSettings" + responses: + "200": + description: migration report + content: + application/json: + schema: + $ref: "#/components/schemas/SecretMigrationReport" + /users/email: get: summary: get current user email (if logged in) @@ -16597,6 +16674,83 @@ components: # -- INLINE END -- # Do not change line above + VaultSettings: + type: object + required: + - address + - mount_path + properties: + address: + type: string + description: HashiCorp Vault server address (e.g., https://vault.company.com:8200) + mount_path: + type: string + description: KV v2 secrets engine mount path (e.g., windmill) + jwt_role: + type: string + description: Vault JWT auth role name for Windmill (optional, if not provided token auth is used) + namespace: + type: string + description: Vault Enterprise namespace (optional) + token: + type: string + description: Static Vault token for testing/development (optional, if provided this is used instead of JWT authentication) + + SecretMigrationFailure: + type: object + required: + - workspace_id + - path + - error + properties: + workspace_id: + type: string + description: Workspace ID where the secret is located + path: + type: string + description: Path of the secret that failed to migrate + error: + type: string + description: Error message + + SecretMigrationReport: + type: object + required: + - total_secrets + - migrated_count + - failed_count + - failures + properties: + total_secrets: + type: integer + format: int64 + description: Total number of secrets found + migrated_count: + type: integer + format: int64 + description: Number of secrets successfully migrated + failed_count: + type: integer + format: int64 + description: Number of secrets that failed to migrate + failures: + type: array + items: + $ref: "#/components/schemas/SecretMigrationFailure" + description: Details of any failures encountered during migration + + JwksResponse: + type: object + required: + - keys + properties: + keys: + type: array + items: + type: object + additionalProperties: true + description: Array of JSON Web Keys for JWT verification + FlowConversation: type: object required: diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index c6f5400f30..7fbfee8c9a 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -149,6 +149,7 @@ mod scim_oss; mod scopes; mod scripts; mod service_logs; +mod secret_backend_ext; mod settings; mod slack_approvals; #[cfg(all(feature = "smtp", feature = "private"))] @@ -719,6 +720,8 @@ pub async fn run_server( .route("/openapi.yaml", get(openapi)) .route("/openapi.json", get(openapi_json)), ) + // JWKS endpoint for HashiCorp Vault JWT authentication (must be outside /api prefix) + .route("/.well-known/jwks.json", get(settings::get_jwks)) .fallback(static_assets::static_handler) .layer(middleware_stack); diff --git a/backend/windmill-api/src/resources.rs b/backend/windmill-api/src/resources.rs index f1f9c471ec..37c34e8feb 100644 --- a/backend/windmill-api/src/resources.rs +++ b/backend/windmill-api/src/resources.rs @@ -10,6 +10,7 @@ use std::collections::HashMap; use crate::{ db::{ApiAuthed, DB}, + secret_backend_ext::rename_vault_secret, users::{maybe_refresh_folders, require_owner_of_path, Tokened}, utils::{check_scopes, require_super_admin, BulkDeleteRequest}, var_resource_cache::{cache_resource, get_cached_resource}, @@ -947,12 +948,40 @@ async fn update_resource( let mut tx = user_db.begin(&authed).await?; - if let Some(npath) = ns.path { + if let Some(npath) = ns.path.clone() { if npath != path { check_path_conflict(&mut tx, &w_id, &npath).await?; require_owner_of_path(&authed, path)?; + // Handle Vault secret rename if the linked variable is a Vault-stored secret + let linked_var = sqlx::query!( + "SELECT value, is_secret FROM variable WHERE path = $1 AND workspace_id = $2", + path, + w_id + ) + .fetch_optional(&mut *tx) + .await?; + + if let Some(var) = linked_var { + if var.is_secret { + // Check if this is a Vault-stored secret and rename it + if let Some(new_value) = + rename_vault_secret(&db, &w_id, path, &npath, &var.value).await? + { + // Update the variable's value to point to the new Vault path + sqlx::query!( + "UPDATE variable SET value = $1 WHERE path = $2 AND workspace_id = $3", + new_value, + path, + w_id + ) + .execute(&mut *tx) + .await?; + } + } + } + sqlx::query!( "UPDATE variable SET path = $1 WHERE path = $2 AND workspace_id = $3", npath, diff --git a/backend/windmill-api/src/secret_backend_ext.rs b/backend/windmill-api/src/secret_backend_ext.rs new file mode 100644 index 0000000000..41d2267a94 --- /dev/null +++ b/backend/windmill-api/src/secret_backend_ext.rs @@ -0,0 +1,408 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Secret backend extension for the API layer +//! +//! This module provides helper functions for integrating the SecretBackend +//! trait with variable operations in the API. +//! +//! Note: HashiCorp Vault integration requires Enterprise Edition. +//! The OSS version only supports the database backend. + +use std::sync::Arc; + +use windmill_common::{ + db::DB, + error::{Error, Result}, + secret_backend::{database::DatabaseBackend, SecretBackend}, + variables::{build_crypt, decrypt, encrypt}, +}; + +#[cfg(all(feature = "private", feature = "enterprise"))] +use windmill_common::{ + global_settings::{load_value_from_global_settings, SECRET_BACKEND_SETTING}, + secret_backend::{SecretBackendConfig, VaultBackend, VaultSettings}, +}; + +#[cfg(all(feature = "private", feature = "enterprise"))] +use tokio::sync::RwLock; + +// Cached Vault backend to avoid recreating it for every request +// This enables connection pooling and avoids repeated setup overhead +#[cfg(all(feature = "private", feature = "enterprise"))] +struct CachedVaultBackend { + backend: Arc, + settings: VaultSettings, +} + +#[cfg(all(feature = "private", feature = "enterprise"))] +lazy_static::lazy_static! { + static ref VAULT_BACKEND_CACHE: RwLock> = RwLock::new(None); +} + +/// Get the current secret backend based on global settings +/// +/// OSS: Always returns DatabaseBackend +/// EE: Returns configured backend (Database or Vault) +#[cfg(not(all(feature = "private", feature = "enterprise")))] +pub async fn get_secret_backend(db: &DB) -> Result> { + Ok(Arc::new(DatabaseBackend::new(db.clone()))) +} + +#[cfg(all(feature = "private", feature = "enterprise"))] +pub async fn get_secret_backend(db: &DB) -> Result> { + let config = match load_value_from_global_settings(db, SECRET_BACKEND_SETTING).await? { + Some(value) => serde_json::from_value::(value).unwrap_or_default(), + None => SecretBackendConfig::default(), + }; + + match config { + SecretBackendConfig::Database => Ok(Arc::new(DatabaseBackend::new(db.clone()))), + SecretBackendConfig::HashiCorpVault(settings) => { + get_or_create_vault_backend(db, settings).await + } + } +} + +/// Get a cached Vault backend or create a new one if settings changed +#[cfg(all(feature = "private", feature = "enterprise"))] +async fn get_or_create_vault_backend( + _db: &DB, + settings: VaultSettings, +) -> Result> { + // Check if we have a cached backend with matching settings (read lock) + { + let cache = VAULT_BACKEND_CACHE.read().await; + if let Some(ref cached) = *cache { + if cached.settings == settings { + return Ok(cached.backend.clone()); + } + } + } + + // Need to create a new backend - acquire write lock + let mut cache = VAULT_BACKEND_CACHE.write().await; + + // Double-check (another task may have created it while we waited) + if let Some(ref cached) = *cache { + if cached.settings == settings { + return Ok(cached.backend.clone()); + } + } + + // Create new backend + let backend: Arc = { + #[cfg(feature = "openidconnect")] + if settings.token.is_none() { + Arc::new(VaultBackend::new_with_db(settings.clone(), _db.clone())) + } else { + Arc::new(VaultBackend::new(settings.clone())) + } + + #[cfg(not(feature = "openidconnect"))] + Arc::new(VaultBackend::new(settings.clone())) + }; + + // Cache it + *cache = Some(CachedVaultBackend { + backend: backend.clone(), + settings, + }); + + Ok(backend) +} + +/// Check if a Vault backend is currently configured +/// +/// OSS: Always returns false +/// EE: Checks global settings +#[cfg(not(all(feature = "private", feature = "enterprise")))] +pub async fn is_vault_backend_configured(_db: &DB) -> Result { + Ok(false) +} + +#[cfg(all(feature = "private", feature = "enterprise"))] +pub async fn is_vault_backend_configured(db: &DB) -> Result { + let config = match load_value_from_global_settings(db, SECRET_BACKEND_SETTING).await? { + Some(value) => serde_json::from_value::(value).unwrap_or_default(), + None => SecretBackendConfig::default(), + }; + + Ok(matches!(config, SecretBackendConfig::HashiCorpVault(_))) +} + +/// Get a secret value using the configured backend +/// +/// For database backend: decrypts using workspace key +/// For vault backend (EE only): fetches from Vault directly +pub async fn get_secret_value( + db: &DB, + workspace_id: &str, + path: &str, + encrypted_value: &str, +) -> Result { + let backend = get_secret_backend(db).await?; + + match backend.backend_name() { + "database" => { + // Use existing database decryption + let mc = build_crypt(db, workspace_id).await?; + decrypt(&mc, encrypted_value.to_string()).map_err(|e| { + Error::internal_err(format!("Error decrypting variable {}: {}", path, e)) + }) + } + "hashicorp_vault" => { + // Fetch from Vault directly + backend.get_secret(workspace_id, path).await + } + _ => Err(Error::internal_err(format!( + "Unknown backend: {}", + backend.backend_name() + ))), + } +} + +/// Store a secret value using the configured backend +/// +/// For database backend: encrypts using workspace key and returns encrypted value +/// For vault backend (EE only): stores in Vault and returns a placeholder for DB storage +pub async fn store_secret_value( + db: &DB, + workspace_id: &str, + path: &str, + plain_value: &str, +) -> Result { + let backend = get_secret_backend(db).await?; + + match backend.backend_name() { + "database" => { + // Use existing database encryption + let mc = build_crypt(db, workspace_id).await?; + Ok(encrypt(&mc, plain_value)) + } + "hashicorp_vault" => { + // Store in Vault and return a marker for DB + backend.set_secret(workspace_id, path, plain_value).await?; + // Return a marker indicating the value is stored in Vault + // The actual value in the DB will be this marker + Ok(format!("$vault:{}", path)) + } + _ => Err(Error::internal_err(format!( + "Unknown backend: {}", + backend.backend_name() + ))), + } +} + +/// Delete a secret from the configured backend (if using Vault) +/// +/// For database backend: no-op (DB delete is handled separately) +/// For vault backend (EE only): deletes from Vault +pub async fn delete_secret_from_backend( + db: &DB, + workspace_id: &str, + path: &str, +) -> Result<()> { + if is_vault_backend_configured(db).await? { + let backend = get_secret_backend(db).await?; + // Ignore NotFound errors during deletion (secret might not exist in Vault) + match backend.delete_secret(workspace_id, path).await { + Ok(()) => Ok(()), + Err(Error::NotFound(_)) => Ok(()), + Err(e) => Err(e), + } + } else { + Ok(()) + } +} + +/// Check if a value is stored in Vault (indicated by the $vault: prefix) +pub fn is_vault_stored_value(value: &str) -> bool { + value.starts_with("$vault:") +} + +/// Rename a secret in Vault when a variable path changes (EE only) +/// +/// This function: +/// 1. Reads the secret value from the old path +/// 2. Writes it to the new path +/// 3. Deletes from the old path +/// 4. Returns the new marker value ($vault:new_path) +/// +/// If the value is not a Vault-stored value, returns None (no action needed). +/// If Vault is not configured, returns None. +#[cfg(not(all(feature = "private", feature = "enterprise")))] +pub async fn rename_vault_secret( + _db: &DB, + _workspace_id: &str, + _old_path: &str, + new_path: &str, + current_value: &str, +) -> Result> { + // OSS: If value has $vault: prefix, just update the reference + // (This handles edge case where EE was used before downgrading to OSS) + if is_vault_stored_value(current_value) { + tracing::warn!( + "Variable has $vault: prefix but Vault requires Enterprise Edition. \ + Updating DB reference to {}", + new_path + ); + return Ok(Some(format!("$vault:{}", new_path))); + } + Ok(None) +} + +#[cfg(all(feature = "private", feature = "enterprise"))] +pub async fn rename_vault_secret( + db: &DB, + workspace_id: &str, + old_path: &str, + new_path: &str, + current_value: &str, +) -> Result> { + // Only handle Vault-stored values + if !is_vault_stored_value(current_value) { + return Ok(None); + } + + // Check if Vault backend is configured + if !is_vault_backend_configured(db).await? { + // Vault not configured but value has $vault: prefix - this is an inconsistent state + // Log warning and return new marker to at least update the DB reference + tracing::warn!( + "Variable value has $vault: prefix but Vault is not configured. \ + Updating DB reference from {} to {}", + old_path, + new_path + ); + return Ok(Some(format!("$vault:{}", new_path))); + } + + let backend = get_secret_backend(db).await?; + + // Read from old path + let secret_value = match backend.get_secret(workspace_id, old_path).await { + Ok(value) => value, + Err(Error::NotFound(_)) => { + // Secret doesn't exist in Vault - just update the DB reference + tracing::warn!( + "Secret not found in Vault at path {} during rename to {}", + old_path, + new_path + ); + return Ok(Some(format!("$vault:{}", new_path))); + } + Err(e) => return Err(e), + }; + + // Write to new path + backend + .set_secret(workspace_id, new_path, &secret_value) + .await?; + + // Delete from old path (ignore errors - new path is already written) + if let Err(e) = backend.delete_secret(workspace_id, old_path).await { + tracing::warn!( + "Failed to delete old secret at {} after rename to {}: {}", + old_path, + new_path, + e + ); + } + + Ok(Some(format!("$vault:{}", new_path))) +} + +/// Bulk rename secrets in Vault when a path prefix changes (e.g., user rename) +/// EE only feature. +/// +/// This is used when renaming users where many secrets need their paths updated. +/// Returns a list of (old_path, new_value) pairs for updating the database. +#[cfg(not(all(feature = "private", feature = "enterprise")))] +pub async fn rename_vault_secrets_with_prefix( + _db: &DB, + _workspace_id: &str, + _old_prefix: &str, + _new_prefix: &str, + _variables: Vec<(String, String)>, +) -> Result> { + // OSS: No Vault support, return empty + Ok(vec![]) +} + +#[cfg(all(feature = "private", feature = "enterprise"))] +pub async fn rename_vault_secrets_with_prefix( + db: &DB, + workspace_id: &str, + old_prefix: &str, + new_prefix: &str, + variables: Vec<(String, String)>, // (path, value) pairs +) -> Result> { + // Only process if Vault is configured + if !is_vault_backend_configured(db).await? { + return Ok(vec![]); + } + + let backend = get_secret_backend(db).await?; + let mut updates = Vec::new(); + + for (old_path, value) in variables { + // Only handle Vault-stored values + if !is_vault_stored_value(&value) { + continue; + } + + // Calculate new path by replacing prefix + let new_path = if old_path.starts_with(old_prefix) { + format!("{}{}", new_prefix, &old_path[old_prefix.len()..]) + } else { + continue; // Path doesn't match prefix, skip + }; + + // Read from old path + let secret_value = match backend.get_secret(workspace_id, &old_path).await { + Ok(v) => v, + Err(Error::NotFound(_)) => { + // Just update DB reference + updates.push((old_path, format!("$vault:{}", new_path))); + continue; + } + Err(e) => { + tracing::error!( + "Failed to read secret at {} during bulk rename: {}", + old_path, + e + ); + continue; + } + }; + + // Write to new path + if let Err(e) = backend.set_secret(workspace_id, &new_path, &secret_value).await { + tracing::error!( + "Failed to write secret to {} during bulk rename: {}", + new_path, + e + ); + continue; + } + + // Delete from old path + if let Err(e) = backend.delete_secret(workspace_id, &old_path).await { + tracing::warn!( + "Failed to delete old secret at {} after rename: {}", + old_path, + e + ); + } + + updates.push((old_path, format!("$vault:{}", new_path))); + } + + Ok(updates) +} diff --git a/backend/windmill-api/src/settings.rs b/backend/windmill-api/src/settings.rs index 939ae9fa57..d6d747a142 100644 --- a/backend/windmill-api/src/settings.rs +++ b/backend/windmill-api/src/settings.rs @@ -42,6 +42,8 @@ use windmill_common::{ }, server::Smtp, }; +#[cfg(all(feature = "private", feature = "enterprise"))] +use windmill_common::secret_backend::{SecretMigrationReport, VaultSettings}; use windmill_common::{error::to_anyhow, PgDatabase}; pub fn global_service() -> Router { @@ -85,6 +87,16 @@ pub fn global_service() -> Router { post(acknowledge_all_critical_alerts), ); + // Vault integration routes (EE only - requires both private and enterprise features) + #[cfg(all(feature = "private", feature = "enterprise"))] + let r = r + .route("/test_secret_backend", post(test_secret_backend)) + .route("/migrate_secrets_to_vault", post(migrate_secrets_to_vault)) + .route( + "/migrate_secrets_to_database", + post(migrate_secrets_to_database), + ); + #[cfg(feature = "parquet")] { return r.route("/test_object_storage_config", post(test_s3_bucket)); @@ -798,3 +810,100 @@ async fn setup_custom_instance_pg_database_inner( Ok(()) } + +// ============================================================================ +// Secret Backend Settings (HashiCorp Vault Integration) - Enterprise Edition +// ============================================================================ + +/// Test connection to a secret backend (HashiCorp Vault) +/// +/// This endpoint validates that the Vault settings are correct and that +/// Windmill can successfully authenticate and communicate with Vault. +/// +/// This is an Enterprise Edition feature. +#[cfg(all(feature = "private", feature = "enterprise"))] +pub async fn test_secret_backend( + Extension(db): Extension, + authed: ApiAuthed, + Json(settings): Json, +) -> Result { + require_super_admin(&db, &authed.email).await?; + + windmill_common::secret_backend::test_vault_connection(&settings, Some(&db)).await?; + + Ok("Successfully connected to HashiCorp Vault".to_string()) +} + +/// Migrate existing secrets from database to HashiCorp Vault +/// +/// This endpoint reads all encrypted secrets from the database, decrypts them, +/// and stores them in HashiCorp Vault. The database values are NOT deleted +/// automatically to allow for rollback if needed. +/// +/// This is an Enterprise Edition feature. +#[cfg(all(feature = "private", feature = "enterprise"))] +pub async fn migrate_secrets_to_vault( + Extension(db): Extension, + authed: ApiAuthed, + Json(settings): Json, +) -> JsonResult { + require_super_admin(&db, &authed.email).await?; + + let report = windmill_common::secret_backend::migrate_secrets_to_vault(&db, &settings).await?; + + Ok(Json(report)) +} + +/// Migrate secrets from HashiCorp Vault back to database +/// +/// This endpoint reads all secrets from HashiCorp Vault, encrypts them using +/// the workspace encryption keys, and stores them in the database. The Vault +/// values are NOT deleted automatically to allow for rollback if needed. +/// +/// This is an Enterprise Edition feature. +#[cfg(all(feature = "private", feature = "enterprise"))] +pub async fn migrate_secrets_to_database( + Extension(db): Extension, + authed: ApiAuthed, + Json(settings): Json, +) -> JsonResult { + require_super_admin(&db, &authed.email).await?; + + let report = + windmill_common::secret_backend::migrate_secrets_to_database(&db, &settings).await?; + + Ok(Json(report)) +} + +// ============================================================================ +// JWKS Endpoint for Vault JWT Authentication +// ============================================================================ + +/// JSON Web Key Set response structure +#[derive(Serialize)] +pub struct JwksResponse { + pub keys: Vec, +} + +/// JWKS endpoint for HashiCorp Vault to validate JWTs +/// +/// Vault calls this endpoint to fetch the public keys used to verify +/// JWTs generated by Windmill for authentication. +/// +/// In the open-source version, this returns an empty JWKS. +/// The Enterprise Edition provides the actual key set. +pub async fn get_jwks() -> JsonResult { + // Open source version returns empty JWKS + // Enterprise Edition will override this with actual public keys + #[cfg(not(feature = "enterprise"))] + { + Ok(Json(JwksResponse { keys: vec![] })) + } + + #[cfg(feature = "enterprise")] + { + // In enterprise mode, the actual keys would be fetched from global settings + // For now, return empty - the EE implementation would override this + Ok(Json(JwksResponse { keys: vec![] })) + } +} diff --git a/backend/windmill-api/src/users.rs b/backend/windmill-api/src/users.rs index 5c03d54646..10cf4cfddd 100644 --- a/backend/windmill-api/src/users.rs +++ b/backend/windmill-api/src/users.rs @@ -19,6 +19,7 @@ use crate::db::ApiAuthed; pub use crate::auth::Tokened; +use crate::secret_backend_ext::rename_vault_secrets_with_prefix; use crate::utils::{ generate_instance_wide_unique_username, get_instance_username_or_create_pending, }; @@ -2661,6 +2662,7 @@ async fn rename_user( } update_username_in_workpsace( &mut tx, + &db, &user_email, &w_u.username, &ru.new_username, @@ -2688,6 +2690,7 @@ async fn rename_user( async fn update_username_in_workpsace<'c>( tx: &mut sqlx::Transaction<'c, sqlx::Postgres>, + db: &DB, email: &str, old_username: &str, new_username: &str, @@ -2805,6 +2808,43 @@ async fn update_username_in_workpsace<'c>( // ---- variables ---- + // Handle Vault secret renames before updating paths in DB + let old_prefix = format!("u/{}/", old_username); + let new_prefix = format!("u/{}/", new_username); + + // Fetch all Vault-stored secret variables under this user's path + let vault_secrets: Vec<(String, String)> = sqlx::query!( + r#"SELECT path, value FROM variable + WHERE path LIKE ('u/' || $1 || '/%') + AND workspace_id = $2 + AND is_secret = true + AND value LIKE '$vault:%'"#, + old_username, + w_id + ) + .fetch_all(&mut **tx) + .await? + .into_iter() + .map(|r| (r.path, r.value)) + .collect(); + + // Rename secrets in Vault and get the new values + let vault_updates = + rename_vault_secrets_with_prefix(db, w_id, &old_prefix, &new_prefix, vault_secrets).await?; + + // Update the values in the DB for renamed Vault secrets (using OLD path, before path update) + for (old_path, new_value) in vault_updates { + sqlx::query!( + "UPDATE variable SET value = $1 WHERE path = $2 AND workspace_id = $3", + new_value, + old_path, + w_id + ) + .execute(&mut **tx) + .await?; + } + + // Now update the paths in the database sqlx::query!( r#"UPDATE variable SET path = REGEXP_REPLACE(path,'u/' || $2 || '/(.*)','u/' || $1 || '/\1') WHERE path LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#, new_username, diff --git a/backend/windmill-api/src/variables.rs b/backend/windmill-api/src/variables.rs index a5493cd2e5..54e68f9a9b 100644 --- a/backend/windmill-api/src/variables.rs +++ b/backend/windmill-api/src/variables.rs @@ -8,6 +8,10 @@ use crate::{ db::{ApiAuthed, DB}, + secret_backend_ext::{ + delete_secret_from_backend, get_secret_value, is_vault_stored_value, rename_vault_secret, + store_secret_value, + }, users::{maybe_refresh_folders, require_owner_of_path}, utils::{check_scopes, BulkDeleteRequest}, webhook_util::{WebhookMessage, WebhookShared}, @@ -39,7 +43,7 @@ use crate::var_resource_cache::{cache_variable, get_cached_variable}; use lazy_static::lazy_static; use serde::Deserialize; use sqlx::{Acquire, Postgres, Transaction}; -use windmill_common::variables::{decrypt, encrypt}; +use windmill_common::variables::encrypt; use windmill_git_sync::{handle_deployment_metadata, DeployedObject}; lazy_static! { @@ -210,13 +214,8 @@ async fn get_variable( return Err(Error::internal_err("Require oauth2 feature".to_string())); } else if !value.is_empty() && decrypt_secret { let _ = tx.commit().await; - let mc = build_crypt(&db, &w_id).await?; - Some(decrypt(&mc, value).map_err(|e| { - Error::internal_err(format!( - "Error decrypting variable {}: {}", - variable.path, e - )) - })?) + // Use secret backend for decryption (supports both DB and Vault) + Some(get_secret_value(&db, &w_id, &variable.path, &value).await?) } else if q.include_encrypted.unwrap_or(false) { Some(value) } else { @@ -355,8 +354,8 @@ async fn create_variable( check_path_conflict(&db, &w_id, &variable.path).await?; let value = if variable.is_secret && !already_encrypted.unwrap_or(false) { - let mc = build_crypt(&db, &w_id).await?; - encrypt(&mc, &variable.value) + // Use secret backend for encryption (supports both DB and Vault) + store_secret_value(&db, &w_id, &variable.path, &variable.value).await? } else { variable.value }; @@ -436,6 +435,16 @@ async fn delete_variable( check_scopes(&authed, || format!("variables:write:{}", path))?; + // Check if variable is a secret before deleting (for Vault cleanup) + let is_secret = sqlx::query_scalar!( + "SELECT is_secret FROM variable WHERE path = $1 AND workspace_id = $2", + path, + &w_id + ) + .fetch_optional(&db) + .await? + .unwrap_or(false); + let mut tx = user_db.begin(&authed).await?; sqlx::query!( @@ -465,6 +474,11 @@ async fn delete_variable( tx.commit().await?; + // If variable was a secret, also delete from Vault backend (if configured) + if is_secret { + delete_secret_from_backend(&db, &w_id, path).await?; + } + handle_deployment_metadata( &authed.email, &authed.username, @@ -496,6 +510,15 @@ async fn delete_variables_bulk( check_scopes(&authed, || format!("variables:write:{}", path))?; } + // Query which paths are secrets before deletion (for Vault cleanup) + let secret_paths: Vec = sqlx::query_scalar!( + "SELECT path FROM variable WHERE path = ANY($1) AND workspace_id = $2 AND is_secret = true", + &request.paths, + &w_id + ) + .fetch_all(&db) + .await?; + let mut tx = user_db.begin(&authed).await?; let deleted_paths = sqlx::query_scalar!( @@ -526,6 +549,13 @@ async fn delete_variables_bulk( tx.commit().await?; + // Delete secrets from Vault backend (if configured) + for path in &secret_paths { + if deleted_paths.contains(path) { + delete_secret_from_backend(&db, &w_id, path).await?; + } + } + try_join_all(deleted_paths.iter().map(|path| { handle_deployment_metadata( &authed.email, @@ -589,7 +619,9 @@ async fn update_variable( sqlb.set_str("path", npath); } let ns_value_is_none = ns.value.is_none(); - if let Some(nvalue) = ns.value { + // Determine the target path for storing secrets (use new path if provided) + let target_path = ns.path.as_deref().unwrap_or(path); + if let Some(nvalue) = ns.value.clone() { let is_secret = if ns.is_secret.is_some() { ns.is_secret.unwrap() } else { @@ -604,8 +636,9 @@ async fn update_variable( }; let value = if is_secret && !already_encrypted.unwrap_or(false) { - let mc = build_crypt(&db, &w_id).await?; - encrypt(&mc, &nvalue) + // Use secret backend for encryption (supports both DB and Vault) + // Store at target_path (new path if renaming, otherwise current path) + store_secret_value(&db, &w_id, target_path, &nvalue).await? } else { nvalue }; @@ -654,11 +687,38 @@ async fn update_variable( let mut tx: Transaction<'_, Postgres> = user_db.begin(&authed).await?; - if let Some(npath) = ns.path { + if let Some(npath) = ns.path.clone() { if npath != path { check_path_conflict(&db, &w_id, &npath).await?; require_owner_of_path(&authed, path)?; + // Handle Vault secret rename if the variable is a secret stored in Vault + let current_var = sqlx::query!( + "SELECT value, is_secret FROM variable WHERE path = $1 AND workspace_id = $2", + path, + w_id + ) + .fetch_optional(&mut *tx) + .await?; + + if let Some(var) = current_var { + if var.is_secret && is_vault_stored_value(&var.value) { + if ns.value.is_some() { + // New value was provided and already stored at new path + // Just delete the old secret from Vault + delete_secret_from_backend(&db, &w_id, path).await?; + } else { + // No new value - rename the secret in Vault + if let Some(new_value) = + rename_vault_secret(&db, &w_id, path, &npath, &var.value).await? + { + // Update the variable's value to point to the new Vault path + sqlb.set_str("value", &new_value); + } + } + } + } + let mut v = sqlx::query_scalar!( "SELECT value FROM resource WHERE path = $1 AND workspace_id = $2", path, @@ -835,13 +895,8 @@ pub async fn get_value_internal<'a>( #[cfg(not(feature = "oauth2"))] return Err(Error::internal_err("Require oauth2 feature".to_string())); } else if !value.is_empty() { - let mc = build_crypt(db_with_opt_authed.db(), &w_id).await?; - decrypt(&mc, value).map_err(|e| { - Error::internal_err(format!( - "Error decrypting variable {}: {}", - variable.path, e - )) - })? + // Use secret backend for decryption (supports both DB and Vault) + get_secret_value(db_with_opt_authed.db(), &w_id, &variable.path, &value).await? } else { "".to_string() } diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index 6dbc8c8ba6..c5428e5c5c 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -47,6 +47,7 @@ pub const JWT_SECRET_SETTING: &str = "jwt_secret"; pub const EMAIL_DOMAIN_SETTING: &str = "email_domain"; pub const OTEL_SETTING: &str = "otel"; pub const APP_WORKSPACED_ROUTE_SETTING: &str = "app_workspaced_route"; +pub const SECRET_BACKEND_SETTING: &str = "secret_backend"; pub const ENV_SETTINGS: &[&str] = &[ "DISABLE_NSJAIL", diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index a650f6bf0c..c61f4263cc 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -78,6 +78,7 @@ pub mod result_stream; pub mod runnable_settings; pub mod s3_helpers; pub mod schedule; +pub mod secret_backend; pub mod schema; pub mod scripts; pub mod server; diff --git a/backend/windmill-common/src/secret_backend/database.rs b/backend/windmill-common/src/secret_backend/database.rs new file mode 100644 index 0000000000..3dee114481 --- /dev/null +++ b/backend/windmill-common/src/secret_backend/database.rs @@ -0,0 +1,126 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Database backend for secret storage +//! +//! This is the default backend that stores secrets encrypted in the PostgreSQL database +//! using the existing magic_crypt encryption with workspace-specific keys. + +use async_trait::async_trait; + +use crate::db::DB; +use crate::error::{Error, Result}; +use crate::variables::{build_crypt, decrypt, encrypt}; + +use super::SecretBackend; + +/// Database-backed secret storage +/// +/// This backend stores secrets encrypted in the `variable` table using +/// the workspace's encryption key. This is the default and original +/// behavior of Windmill. +pub struct DatabaseBackend { + db: DB, +} + +impl DatabaseBackend { + /// Create a new database backend + pub fn new(db: DB) -> Self { + Self { db } + } +} + +#[async_trait] +impl SecretBackend for DatabaseBackend { + async fn get_secret(&self, workspace_id: &str, path: &str) -> Result { + let variable = sqlx::query!( + "SELECT value FROM variable WHERE path = $1 AND workspace_id = $2 AND is_secret = true", + path, + workspace_id + ) + .fetch_optional(&self.db) + .await?; + + let variable = variable.ok_or_else(|| { + Error::NotFound(format!( + "Secret variable {} not found in workspace {}", + path, workspace_id + )) + })?; + + let value = variable.value; + if value.is_empty() { + return Ok(String::new()); + } + + let mc = build_crypt(&self.db, workspace_id).await?; + decrypt(&mc, value).map_err(|e| { + Error::internal_err(format!("Error decrypting variable {}: {}", path, e)) + }) + } + + async fn set_secret(&self, workspace_id: &str, path: &str, value: &str) -> Result<()> { + let mc = build_crypt(&self.db, workspace_id).await?; + let encrypted_value = encrypt(&mc, value); + + // Update the value in the database + // Note: This assumes the variable row already exists (created via the normal API) + let result = sqlx::query!( + "UPDATE variable SET value = $1 WHERE path = $2 AND workspace_id = $3 AND is_secret = true", + encrypted_value, + path, + workspace_id + ) + .execute(&self.db) + .await?; + + if result.rows_affected() == 0 { + return Err(Error::NotFound(format!( + "Secret variable {} not found in workspace {}", + path, workspace_id + ))); + } + + Ok(()) + } + + async fn delete_secret(&self, _workspace_id: &str, _path: &str) -> Result<()> { + // For database backend, deletion is handled by the normal variable deletion flow + // The encrypted value is just deleted along with the row + // This method is a no-op for database backend since the caller handles the DELETE + Ok(()) + } + + fn backend_name(&self) -> &'static str { + "database" + } +} + +/// Encrypt a value for storage in the database +/// +/// This is a convenience function for use when creating new secrets. +pub async fn encrypt_for_database(db: &DB, workspace_id: &str, value: &str) -> Result { + let mc = build_crypt(db, workspace_id).await?; + Ok(encrypt(&mc, value)) +} + +/// Decrypt a value from the database +/// +/// This is a convenience function for reading secrets. +pub async fn decrypt_from_database( + db: &DB, + workspace_id: &str, + encrypted_value: String, +) -> Result { + if encrypted_value.is_empty() { + return Ok(String::new()); + } + let mc = build_crypt(db, workspace_id).await?; + decrypt(&mc, encrypted_value) + .map_err(|e| Error::internal_err(format!("Error decrypting value: {}", e))) +} diff --git a/backend/windmill-common/src/secret_backend/mod.rs b/backend/windmill-common/src/secret_backend/mod.rs new file mode 100644 index 0000000000..772a5ebe6a --- /dev/null +++ b/backend/windmill-common/src/secret_backend/mod.rs @@ -0,0 +1,133 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Secret Backend abstraction for storing secrets in external vaults +//! +//! This module provides a trait-based abstraction for secret storage, +//! allowing secrets to be stored in the database (default) or in external +//! vaults like HashiCorp Vault (Enterprise Edition). + +pub mod database; + +#[cfg(feature = "private")] +pub mod vault_ee; + +pub mod vault_oss; + +#[cfg(test)] +mod tests; + +#[cfg(feature = "private")] +pub use vault_ee::*; + +#[cfg(not(feature = "private"))] +pub use vault_oss::*; + +use async_trait::async_trait; +use serde::{Deserialize, Serialize}; + +use crate::error::Result; + +/// Trait for secret storage backends +/// +/// Implementations of this trait handle the storage and retrieval of secrets. +/// The default implementation stores secrets encrypted in the database. +/// Enterprise Edition supports HashiCorp Vault as an alternative backend. +#[async_trait] +pub trait SecretBackend: Send + Sync { + /// Retrieve a secret value + /// + /// # Arguments + /// * `workspace_id` - The workspace identifier + /// * `path` - The path/name of the secret variable + /// + /// # Returns + /// The decrypted secret value + async fn get_secret(&self, workspace_id: &str, path: &str) -> Result; + + /// Store a secret value + /// + /// # Arguments + /// * `workspace_id` - The workspace identifier + /// * `path` - The path/name of the secret variable + /// * `value` - The plaintext secret value to store + async fn set_secret(&self, workspace_id: &str, path: &str, value: &str) -> Result<()>; + + /// Delete a secret + /// + /// # Arguments + /// * `workspace_id` - The workspace identifier + /// * `path` - The path/name of the secret variable + async fn delete_secret(&self, workspace_id: &str, path: &str) -> Result<()>; + + /// Get the name of this backend for logging/debugging + fn backend_name(&self) -> &'static str; +} + +/// Configuration for secret storage backend +/// +/// This enum is stored in global_settings and determines which backend +/// is used for secret storage at the instance level. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(tag = "type")] +pub enum SecretBackendConfig { + /// Store secrets encrypted in the database (default behavior) + Database, + /// Store secrets in HashiCorp Vault (Enterprise Edition only) + HashiCorpVault(VaultSettings), +} + +impl Default for SecretBackendConfig { + fn default() -> Self { + SecretBackendConfig::Database + } +} + +/// Settings for HashiCorp Vault integration +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct VaultSettings { + /// Vault server address (e.g., "https://vault.company.com:8200") + pub address: String, + /// KV v2 mount path (e.g., "windmill") + pub mount_path: String, + /// JWT auth role name configured in Vault (used for JWT/OIDC auth) + /// Optional - if not provided, token auth is used + #[serde(skip_serializing_if = "Option::is_none")] + pub jwt_role: Option, + /// Vault Enterprise namespace (optional) + #[serde(skip_serializing_if = "Option::is_none")] + pub namespace: Option, + /// Static Vault token for testing/development (optional) + /// If provided, this is used instead of JWT authentication + #[serde(skip_serializing_if = "Option::is_none")] + pub token: Option, +} + +/// Result of a secret migration operation +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SecretMigrationReport { + /// Total number of secrets found + pub total_secrets: usize, + /// Number of secrets successfully migrated + pub migrated_count: usize, + /// Number of secrets that failed to migrate + pub failed_count: usize, + /// Details of any failures + pub failures: Vec, +} + +/// Details of a failed secret migration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SecretMigrationFailure { + /// Workspace ID where the secret is located + pub workspace_id: String, + /// Path of the secret that failed to migrate + pub path: String, + /// Error message + pub error: String, +} diff --git a/backend/windmill-common/src/secret_backend/tests.rs b/backend/windmill-common/src/secret_backend/tests.rs new file mode 100644 index 0000000000..2a6962f8dd --- /dev/null +++ b/backend/windmill-common/src/secret_backend/tests.rs @@ -0,0 +1,143 @@ +/* + * Integration tests for secret backend + * + * These tests require a running Vault instance at http://127.0.0.1:8200 + * with the token "test-root-token" and a KV v2 mount at "windmill". + * + * Run Vault dev server: + * podman run -d --name vault-test --rm -p 8200:8200 \ + * -e 'VAULT_DEV_ROOT_TOKEN_ID=test-root-token' \ + * -e 'VAULT_DEV_LISTEN_ADDRESS=0.0.0.0:8200' \ + * docker.io/hashicorp/vault:latest + * + * Then enable the windmill mount: + * curl --header "X-Vault-Token: test-root-token" \ + * --request POST \ + * --data '{"type": "kv", "options": {"version": "2"}}' \ + * http://127.0.0.1:8200/v1/sys/mounts/windmill + */ + +#[cfg(test)] +mod tests { + use crate::secret_backend::{SecretBackend, VaultBackend, VaultSettings}; + + fn test_settings() -> VaultSettings { + VaultSettings { + address: "http://127.0.0.1:8200".to_string(), + mount_path: "windmill".to_string(), + jwt_role: Some("windmill-secrets".to_string()), + namespace: None, + token: Some("test-root-token".to_string()), + } + } + + #[tokio::test] + #[ignore] // Run with --ignored to execute + async fn test_vault_write_and_read() { + let settings = test_settings(); + let backend = VaultBackend::new(settings); + + let workspace_id = "test-ws-1"; + let path = "test-secret"; + let value = "my-super-secret-value"; + + // Write secret + backend + .set_secret(workspace_id, path, value) + .await + .expect("Failed to write secret"); + + // Read secret + let read_value = backend + .get_secret(workspace_id, path) + .await + .expect("Failed to read secret"); + + assert_eq!(read_value, value); + + // Delete secret + backend + .delete_secret(workspace_id, path) + .await + .expect("Failed to delete secret"); + + // Verify deleted + let result = backend.get_secret(workspace_id, path).await; + assert!(result.is_err(), "Secret should be deleted"); + } + + #[tokio::test] + #[ignore] + async fn test_vault_multiple_secrets() { + let settings = test_settings(); + let backend = VaultBackend::new(settings); + + let workspace_id = "test-ws-2"; + + // Write multiple secrets + for i in 0..5 { + let path = format!("secret-{}", i); + let value = format!("value-{}", i); + backend + .set_secret(workspace_id, &path, &value) + .await + .expect(&format!("Failed to write secret-{}", i)); + } + + // Read and verify + for i in 0..5 { + let path = format!("secret-{}", i); + let expected = format!("value-{}", i); + let value = backend + .get_secret(workspace_id, &path) + .await + .expect(&format!("Failed to read secret-{}", i)); + assert_eq!(value, expected); + } + + // Cleanup + for i in 0..5 { + let path = format!("secret-{}", i); + backend + .delete_secret(workspace_id, &path) + .await + .expect(&format!("Failed to delete secret-{}", i)); + } + } + + #[tokio::test] + #[ignore] + async fn test_vault_overwrite_secret() { + let settings = test_settings(); + let backend = VaultBackend::new(settings); + + let workspace_id = "test-ws-3"; + let path = "overwrite-test"; + + // Write initial value + backend + .set_secret(workspace_id, path, "initial-value") + .await + .expect("Failed to write initial value"); + + // Overwrite + backend + .set_secret(workspace_id, path, "new-value") + .await + .expect("Failed to overwrite"); + + // Read and verify + let value = backend + .get_secret(workspace_id, path) + .await + .expect("Failed to read"); + + assert_eq!(value, "new-value"); + + // Cleanup + backend + .delete_secret(workspace_id, path) + .await + .expect("Failed to delete"); + } +} diff --git a/backend/windmill-common/src/secret_backend/vault_oss.rs b/backend/windmill-common/src/secret_backend/vault_oss.rs new file mode 100644 index 0000000000..b338388312 --- /dev/null +++ b/backend/windmill-common/src/secret_backend/vault_oss.rs @@ -0,0 +1,107 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! HashiCorp Vault secret backend stubs (Open Source Edition) +//! +//! This module provides stub implementations for Vault integration. +//! The actual Vault integration requires Enterprise Edition. + +use std::sync::Arc; + +use crate::db::DB; +use crate::error::{Error, Result}; + +use super::{database::DatabaseBackend, SecretBackend, SecretBackendConfig, SecretMigrationReport, VaultSettings}; + +/// Stub VaultBackend for OSS - all operations return EE required error +pub struct VaultBackend; + +impl VaultBackend { + pub fn new(_settings: VaultSettings) -> Self { + Self + } +} + +#[async_trait::async_trait] +impl SecretBackend for VaultBackend { + async fn get_secret(&self, _workspace_id: &str, _path: &str) -> Result { + Err(Error::internal_err( + "HashiCorp Vault integration requires Enterprise Edition".to_string(), + )) + } + + async fn set_secret(&self, _workspace_id: &str, _path: &str, _value: &str) -> Result<()> { + Err(Error::internal_err( + "HashiCorp Vault integration requires Enterprise Edition".to_string(), + )) + } + + async fn delete_secret(&self, _workspace_id: &str, _path: &str) -> Result<()> { + Err(Error::internal_err( + "HashiCorp Vault integration requires Enterprise Edition".to_string(), + )) + } + + fn backend_name(&self) -> &'static str { + "hashicorp_vault" + } +} + +/// Create the appropriate secret backend based on configuration +/// +/// In OSS, always returns DatabaseBackend regardless of config. +/// Vault configuration is ignored with a warning. +pub async fn create_secret_backend( + db: DB, + config: &SecretBackendConfig, +) -> Result> { + match config { + SecretBackendConfig::Database => Ok(Arc::new(DatabaseBackend::new(db))), + SecretBackendConfig::HashiCorpVault(_) => { + tracing::warn!( + "HashiCorp Vault is configured but requires Enterprise Edition. \ + Falling back to database backend." + ); + Ok(Arc::new(DatabaseBackend::new(db))) + } + } +} + +/// Test connection to Vault (OSS stub) +pub async fn test_vault_connection(_settings: &VaultSettings, _db: Option<&DB>) -> Result<()> { + Err(Error::internal_err( + "HashiCorp Vault integration requires Enterprise Edition".to_string(), + )) +} + +/// Migrate secrets from database to Vault (OSS stub) +pub async fn migrate_secrets_to_vault( + _db: &DB, + _settings: &VaultSettings, +) -> Result { + Err(Error::internal_err( + "HashiCorp Vault integration requires Enterprise Edition".to_string(), + )) +} + +/// Migrate secrets from Vault back to database (OSS stub) +pub async fn migrate_secrets_to_database( + _db: &DB, + _settings: &VaultSettings, +) -> Result { + Err(Error::internal_err( + "HashiCorp Vault integration requires Enterprise Edition".to_string(), + )) +} + +/// Generate a JWT for Vault authentication (OSS stub) +pub async fn generate_vault_jwt(_db: &DB, _vault_address: &str) -> Result { + Err(Error::internal_err( + "HashiCorp Vault integration requires Enterprise Edition".to_string(), + )) +} diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 8a8ad6f15e..13c9d67dde 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -835,7 +835,6 @@ "version": "1.7.1", "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.7.1.tgz", "integrity": "sha512-o1uhUASyo921r2XtHYOHy7gdkGLge8ghBEQHMWmyJFoXlpU58kIrhhN3w26lpQb6dspetweapMn2CSNwQ8I4wg==", - "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -847,7 +846,6 @@ "version": "1.7.1", "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.7.1.tgz", "integrity": "sha512-PVtJr5CmLwYAU9PZDMITZoR5iAOShYREoR45EyyLrbntV50mdePTgUn4AmOw90Ifcj+x2kRjdzr1HP3RrNiHGA==", - "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -858,7 +856,6 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.1.0.tgz", "integrity": "sha512-WI0DdZ8xFSbgMjR1sFsKABJ/C5OnRrjT06JXbZKexJGrDuPTzZdDYfFlsgcCXCyf+suG5QU2e/y1Wo2V/OapLQ==", - "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -1348,7 +1345,6 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.0.tgz", "integrity": "sha512-Fq6DJW+Bb5jaWE69/qOE0D1TUN9+6uWhCeZpdnSBk14pjLcCWR7Q8n49PTSPHazM37JqrsdpEthXy2xn6jWWiA==", - "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -1550,7 +1546,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1567,7 +1562,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1584,7 +1578,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1601,7 +1594,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1618,7 +1610,6 @@ "cpu": [ "arm" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1635,7 +1626,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1652,7 +1642,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1669,7 +1658,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1686,7 +1674,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1703,7 +1690,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1720,7 +1706,6 @@ "cpu": [ "wasm32" ], - "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -1737,7 +1722,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1754,7 +1738,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -2375,7 +2358,6 @@ "version": "0.10.1", "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.1.tgz", "integrity": "sha512-9tTaPJLSiejZKx+Bmog4uSubteqTvFrVrURwkmHixBo0G4seD0zUxp98E1DzUBJxLQ3NPwXrGKDiVjwx/DpPsg==", - "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -7214,7 +7196,7 @@ "version": "1.21.7", "resolved": "https://registry.npmjs.org/jiti/-/jiti-1.21.7.tgz", "integrity": "sha512-/imKNG4EbWNrVjoNC/1H5/9GFy+tqjGBHCaSsN+P2RnPqjsLmv6UD3Ej+Kj8nBWaRAwyk7kK5ZUc+OEatnTR3A==", - "dev": true, + "devOptional": true, "license": "MIT", "bin": { "jiti": "bin/jiti.js" @@ -7650,7 +7632,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7671,7 +7652,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7692,7 +7672,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7713,7 +7692,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7734,7 +7712,6 @@ "cpu": [ "arm" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7755,7 +7732,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7776,7 +7752,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7797,7 +7772,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7818,7 +7792,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7839,7 +7812,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7860,7 +7832,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -12498,21 +12469,6 @@ } } }, - "node_modules/svelte-check/node_modules/picomatch": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", - "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, "node_modules/svelte-eslint-parser": { "version": "0.43.0", "resolved": "https://registry.npmjs.org/svelte-eslint-parser/-/svelte-eslint-parser-0.43.0.tgz", diff --git a/frontend/src/lib/components/InstanceSetting.svelte b/frontend/src/lib/components/InstanceSetting.svelte index b1920a04ff..42a77b88dd 100644 --- a/frontend/src/lib/components/InstanceSetting.svelte +++ b/frontend/src/lib/components/InstanceSetting.svelte @@ -28,6 +28,7 @@ import EEOnly from './EEOnly.svelte' import CriticalAlertChannels from './instanceSettings/CriticalAlertChannels.svelte' import SmtpSettings from './instanceSettings/SmtpSettings.svelte' + import SecretBackendConfig from './instanceSettings/SecretBackendConfig.svelte' import TextInput from './text_input/TextInput.svelte' import Label from './Label.svelte' @@ -886,6 +887,8 @@ TODO {:else if setting.fieldType == 'smtp_connect'} + {:else if setting.fieldType == 'secret_backend'} + {/if} {#if hasError} diff --git a/frontend/src/lib/components/instanceSettings.ts b/frontend/src/lib/components/instanceSettings.ts index e8c5bff667..cb8b38f95f 100644 --- a/frontend/src/lib/components/instanceSettings.ts +++ b/frontend/src/lib/components/instanceSettings.ts @@ -34,6 +34,7 @@ export interface Setting { | 'smtp_connect' | 'indexer_rates' | 'otel' + | 'secret_backend' storage: SettingStorage advancedToggle?: { label: string @@ -473,6 +474,17 @@ export const settings: Record = { fieldType: 'boolean', storage: 'setting' } + ], + 'Secret Storage': [ + { + label: 'Secret Storage Backend', + description: + 'Configure where secrets (secret variables) are stored. By default, secrets are encrypted and stored in the database. Enterprise Edition supports HashiCorp Vault as an external secret store.', + key: 'secret_backend', + fieldType: 'secret_backend', + storage: 'setting', + ee_only: 'HashiCorp Vault integration is an Enterprise Edition feature' + } ] } diff --git a/frontend/src/lib/components/instanceSettings/SecretBackendConfig.svelte b/frontend/src/lib/components/instanceSettings/SecretBackendConfig.svelte new file mode 100644 index 0000000000..c66876ad65 --- /dev/null +++ b/frontend/src/lib/components/instanceSettings/SecretBackendConfig.svelte @@ -0,0 +1,539 @@ + + +
+ +
+ + setBackendType(v)} + > + {#snippet children({ item: toggleButton })} + + + {/snippet} + + {#if vaultDisabled} +
+ HashiCorp Vault integration requires Enterprise Edition +
+ {/if} +
+ + {#if selectedType === 'Database'} +
+ +
+

Database Storage (Default)

+

+ Secrets are encrypted using workspace-specific keys and stored in the PostgreSQL database. +

+
+
+ {:else if selectedType === 'HashiCorpVault'} + +
+
+ +
+

+ HashiCorp Vault Configuration + Beta +

+

+ Store secrets in an external HashiCorp Vault instance. +

+
+
+ +
+
+ + +
+ +
+ + The KV v2 secrets engine mount path in Vault + +
+ + +
+ + setAuthMethod(v)} + > + {#snippet children({ item: toggleButton })} + + + {/snippet} + +
+ + {#if authMethod === 'token'} +
+ + Static token for authentication. Recommended only for testing/development. + +
+ {:else} +
+ + The JWT authentication role configured in Vault. + + + +
+ Vault JWT Setup Instructions +
+

Configure Vault to accept JWTs from Windmill:

+
+ 
# Enable JWT auth method
+vault auth enable jwt
+
+# Configure JWT auth with Windmill's JWKS endpoint
+vault write auth/jwt/config \
+  jwks_url="{baseUrl}/.well-known/jwks.json" \
+  bound_issuer="{baseUrl}"
+
+# Create a policy for Windmill secrets
+vault policy write windmill-secrets - <<EOF
+path "windmill/data/*" {
+  capabilities = ["create", "read", "update", "delete"]
+}
+path "windmill/metadata/*" {
+  capabilities = ["list", "delete"]
+}
+EOF
+
+# Create the JWT role
+vault write auth/jwt/role/windmill-secrets \
+  role_type="jwt" \
+  bound_audiences="{baseUrl}" \
+  user_claim="email" \
+  policies="windmill-secrets" \
+  ttl="1h"
+
+

+ Replace windmill-secrets with your role name if different. +

+
+
+
+ {/if} + +
+ + Vault Enterprise namespace (leave empty if not using namespaces) + +
+
+ + +
+
+ +
+ + +
+ + + Migrate secrets between the database and HashiCorp Vault. Original values are NOT + deleted to allow for rollback. + + +
+ +
+
+ + + +
+

Database → Vault

+

+ Decrypt secrets from database and store in Vault +

+ +
+ + +
+
+ + + +
+

Vault → Database

+

+ Read secrets from Vault and encrypt in database +

+ +
+
+
+
+
+ {/if} +
+ + + { + migrateToVaultModalOpen = false + }} + onConfirmed={migrateSecretsToVault} +> + {#snippet children()} +
+

+ This will migrate all existing secrets from the database to HashiCorp Vault. The process + will: +

+
    +
  1. Read all encrypted secrets from the database
    2. +
  2. Decrypt them using the workspace encryption keys
    3. +
  3. Store them in HashiCorp Vault under the configured mount path
    4. +
+

+ Note: Database values are NOT deleted automatically. You can manually clear them after + verifying the migration was successful. +

+

Are you sure you want to proceed?

+
+ {/snippet} + + + + { + migrateToDatabaseModalOpen = false + }} + onConfirmed={migrateSecretsToDatabase} +> + {#snippet children()} +
+

+ This will migrate all secrets from HashiCorp Vault back to the database. The process will: +

+
    +
  1. List all secrets in Vault for each workspace
    2. +
  2. Read each secret value from Vault
    3. +
  3. Encrypt and store them in the database
    4. +
+

+ Note: Vault values are NOT deleted automatically. Only secrets that already exist in the + database will be updated. +

+

Are you sure you want to proceed?

+
+ {/snippet} + From 83cf1d3d9098fdc551372bd64adddf2b0db97cd5 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sun, 18 Jan 2026 23:49:09 +0000 Subject: [PATCH 15/19] fix compile --- backend/ee-repo-ref.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 60e3437748..130e307a08 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -54a01bfeb32cedb779c8d39ae2fbfd26c8be0482 \ No newline at end of file +a4790328e8fb78256b6f746a8825ec6ec499c3a8 \ No newline at end of file From 4f8110eb9852b78b48aabbed114c75cbf0d1a2ef Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 19 Jan 2026 08:17:00 +0000 Subject: [PATCH 16/19] fix(flow-chat): handle SSE timeout and fix temp message race condition - Add proper handling for SSE timeout, ping, error, and not_found message types in handleStreamingMessage. On timeout after 30s, the connection now properly closes and reconnects instead of silently failing. - Fix race condition where the first text bubble would disappear during streaming. The pollConversationMessages method was removing all temp messages on every poll interval, even while streaming was active. Now temp messages are only removed during the final poll after job completion. Co-Authored-By: Claude Opus 4.5 --- .../conversations/FlowChatManager.svelte.ts | 67 ++++++++++++++++--- 1 file changed, 59 insertions(+), 8 deletions(-) diff --git a/frontend/src/lib/components/flows/conversations/FlowChatManager.svelte.ts b/frontend/src/lib/components/flows/conversations/FlowChatManager.svelte.ts index ddd0aad1c8..d91b27d4c0 100644 --- a/frontend/src/lib/components/flows/conversations/FlowChatManager.svelte.ts +++ b/frontend/src/lib/components/flows/conversations/FlowChatManager.svelte.ts @@ -322,14 +322,19 @@ export class FlowChatManager { // Do a final poll to get all messages from database try { if (this.selectedConversationId) { - await this.pollConversationMessages(this.selectedConversationId) + await this.pollConversationMessages(this.selectedConversationId, { + removeTempMessages: true + }) } } catch {} this.cleanup() } } - private async pollConversationMessages(conversationId: string, isNewConversation?: boolean) { + private async pollConversationMessages( + conversationId: string, + options?: { isNewConversation?: boolean; removeTempMessages?: boolean } + ) { if (!get(workspaceStore)) return try { @@ -342,7 +347,7 @@ export class FlowChatManager { afterId: lastId }) - if (isNewConversation) { + if (options?.isNewConversation) { await this.refreshConversations() } @@ -355,8 +360,11 @@ export class FlowChatManager { } } - // Remove temporary messages - this.messages = this.messages.filter((msg) => !msg.id.startsWith('temp-')) + // Only remove temporary messages when explicitly requested (e.g., after job completion) + // During streaming, we keep temp messages to avoid them disappearing due to race conditions + if (options?.removeTempMessages) { + this.messages = this.messages.filter((msg) => !msg.id.startsWith('temp-')) + } } catch (error) { console.error('Polling error:', error) } @@ -365,7 +373,7 @@ export class FlowChatManager { private startPolling(conversationId: string, isNewConversation?: boolean) { if (this.pollingInterval) return this.pollingInterval = setInterval(() => { - this.pollConversationMessages(conversationId, isNewConversation) + this.pollConversationMessages(conversationId, { isNewConversation }) }, 500) // Poll every 0.5 seconds setTimeout( () => { @@ -491,7 +499,48 @@ export class FlowChatManager { eventSource.onmessage = async (event) => { try { const data = JSON.parse(event.data) - if (data.type === 'update') { + const type = data.type + + // Handle timeout - reconnect to SSE + if (type === 'timeout') { + eventSource.close() + this.currentEventSource = undefined + // Reconnect + this.handleStreamingMessage( + messageContent, + currentConversationId, + isNewConversation, + additionalInputs + ) + return + } + + // Handle ping - just ignore + if (type === 'ping') { + return + } + + // Handle error + if (type === 'error') { + eventSource.close() + this.currentEventSource = undefined + console.error('SSE error:', data) + sendUserToast('Stream error: ' + (data.error || 'Unknown error'), true) + this.cleanup() + return + } + + // Handle not found + if (type === 'not_found') { + eventSource.close() + this.currentEventSource = undefined + console.error('Job not found') + sendUserToast('Job not found', true) + this.cleanup() + return + } + + if (type === 'update') { if (data.flow_stream_job_id) { this.currentJobId = data.flow_stream_job_id } @@ -567,7 +616,9 @@ export class FlowChatManager { isCompleted = true // Do a final poll to get all messages from database if (this.selectedConversationId) { - await this.pollConversationMessages(this.selectedConversationId) + await this.pollConversationMessages(this.selectedConversationId, { + removeTempMessages: true + }) } this.cleanup() } From 1c8c7949feea3f02dd1d6595c7de6650f756d1a3 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 19 Jan 2026 10:59:42 +0000 Subject: [PATCH 17/19] chore(main): release 1.611.0 (#7604) * chore(main): release 1.611.0 * Apply automatic changes --------- Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com> --- CHANGELOG.md | 12 + backend/Cargo.lock | 215 +++++++++--------- backend/Cargo.toml | 4 +- backend/windmill-api/openapi.yaml | 2 +- benchmarks/lib.ts | 2 +- cli/src/main.ts | 2 +- frontend/package-lock.json | 50 +++- frontend/package.json | 2 +- lsp/Pipfile | 4 +- openflow.openapi.yaml | 2 +- .../WindmillClient/WindmillClient.psd1 | 2 +- python-client/wmill/pyproject.toml | 2 +- python-client/wmill_pg/pyproject.toml | 2 +- typescript-client/jsr.json | 2 +- typescript-client/package.json | 2 +- version.txt | 2 +- 16 files changed, 181 insertions(+), 126 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ca2d357e9..594fdac819 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## [1.611.0](https://github.com/windmill-labs/windmill/compare/v1.610.1...v1.611.0) (2026-01-19) + + +### Features + +* add HashiCorp Vault secret storage integration ([#7599](https://github.com/windmill-labs/windmill/issues/7599)) ([1b9d1c5](https://github.com/windmill-labs/windmill/commit/1b9d1c56c7e49042677326eb397e10d34a3ddcdf)) + + +### Bug Fixes + +* **flow-chat:** handle SSE timeout and fix temp message race condition ([4f8110e](https://github.com/windmill-labs/windmill/commit/4f8110eb9852b78b48aabbed114c75cbf0d1a2ef)) + ## [1.610.1](https://github.com/windmill-labs/windmill/compare/v1.610.0...v1.610.1) (2026-01-17) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 78d498d2b3..7981eebf65 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -1716,7 +1716,7 @@ dependencies = [ "serde_json", "serde_repr", "serde_urlencoded", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "tokio-util", "tower-service", @@ -3557,7 +3557,7 @@ dependencies = [ "swc_visit", "swc_visit_macros", "text_lines", - "thiserror 2.0.17", + "thiserror 2.0.18", "unicode-width 0.1.14", "url", ] @@ -3571,7 +3571,7 @@ dependencies = [ "async-trait", "deno_core", "deno_error", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "uuid", ] @@ -3588,7 +3588,7 @@ dependencies = [ "rusqlite", "serde", "sha2 0.10.9", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", ] @@ -3632,7 +3632,7 @@ dependencies = [ "deno_webgpu", "image", "serde", - "thiserror 2.0.17", + "thiserror 2.0.18", ] [[package]] @@ -3658,7 +3658,7 @@ dependencies = [ "serde", "serde_json", "sys_traits", - "thiserror 2.0.17", + "thiserror 2.0.18", "url", ] @@ -3703,7 +3703,7 @@ dependencies = [ "smallvec", "sourcemap 8.0.1", "static_assertions", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "url", "v8", @@ -3728,7 +3728,7 @@ dependencies = [ "deno_core", "deno_error", "saffron", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", ] @@ -3766,7 +3766,7 @@ dependencies = [ "sha2 0.10.9", "signature", "spki", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "uuid", "x25519-dalek", @@ -3826,7 +3826,7 @@ dependencies = [ "rustls-webpki 0.102.8", "serde", "serde_json", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "tokio-rustls 0.26.4", "tokio-socks", @@ -3854,7 +3854,7 @@ dependencies = [ "serde", "serde-value", "serde_json", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "winapi", ] @@ -3880,7 +3880,7 @@ dependencies = [ "rand 0.8.5", "rayon", "serde", - "thiserror 2.0.17", + "thiserror 2.0.18", "winapi", "windows-sys 0.59.0", ] @@ -3919,7 +3919,7 @@ dependencies = [ "scopeguard", "serde", "smallvec", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "tokio-util", ] @@ -3979,7 +3979,7 @@ dependencies = [ "rand 0.8.5", "rusqlite", "serde", - "thiserror 2.0.17", + "thiserror 2.0.18", "url", ] @@ -3992,7 +3992,7 @@ dependencies = [ "deno_semver", "serde", "serde_json", - "thiserror 2.0.17", + "thiserror 2.0.18", ] [[package]] @@ -4019,7 +4019,7 @@ dependencies = [ "libloading 0.7.4", "log", "napi_sym", - "thiserror 2.0.17", + "thiserror 2.0.18", "windows-sys 0.59.0", ] @@ -4053,7 +4053,7 @@ dependencies = [ "rustls-tokio-stream", "serde", "socket2 0.5.10", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", ] @@ -4140,7 +4140,7 @@ dependencies = [ "spki", "stable_deref_trait", "sys_traits", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "tokio-eld", "url", @@ -4168,7 +4168,7 @@ dependencies = [ "monch", "serde", "serde_json", - "thiserror 2.0.17", + "thiserror 2.0.18", "url", ] @@ -4186,7 +4186,7 @@ dependencies = [ "strum 0.25.0", "strum_macros 0.25.3", "syn 2.0.114", - "thiserror 2.0.17", + "thiserror 2.0.18", ] [[package]] @@ -4207,7 +4207,7 @@ dependencies = [ "serde", "signal-hook", "signal-hook-registry", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "winapi", ] @@ -4226,7 +4226,7 @@ dependencies = [ "serde", "serde_json", "sys_traits", - "thiserror 2.0.17", + "thiserror 2.0.18", "url", ] @@ -4239,7 +4239,7 @@ dependencies = [ "deno_error", "percent-encoding", "sys_traits", - "thiserror 2.0.17", + "thiserror 2.0.18", "url", ] @@ -4260,7 +4260,7 @@ dependencies = [ "once_cell", "percent-encoding", "serde", - "thiserror 2.0.17", + "thiserror 2.0.18", "which 6.0.3", "winapi", ] @@ -4287,7 +4287,7 @@ dependencies = [ "serde", "simd-json", "tempfile", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "which 6.0.3", "winapi", @@ -4321,7 +4321,7 @@ dependencies = [ "once_cell", "parking_lot", "sys_traits", - "thiserror 2.0.17", + "thiserror 2.0.18", "url", ] @@ -4386,7 +4386,7 @@ dependencies = [ "serde", "sys_traits", "tempfile", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "tokio-metrics", "twox-hash 1.6.3", @@ -4409,7 +4409,7 @@ dependencies = [ "monch", "once_cell", "serde", - "thiserror 2.0.17", + "thiserror 2.0.18", "url", ] @@ -4436,7 +4436,7 @@ dependencies = [ "opentelemetry_sdk", "pin-project", "serde", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", ] @@ -4464,7 +4464,7 @@ dependencies = [ "rustls-tokio-stream", "rustls-webpki 0.102.8", "serde", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "webpki-roots 0.26.11", ] @@ -4488,7 +4488,7 @@ checksum = "d79e743ad841f7826d46c6944580f5ba665fe9ab4c31a68c4eed8b5a78225da3" dependencies = [ "deno_core", "deno_error", - "thiserror 2.0.17", + "thiserror 2.0.18", "urlpattern", ] @@ -4508,7 +4508,7 @@ dependencies = [ "flate2", "futures", "serde", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "uuid", ] @@ -4523,7 +4523,7 @@ dependencies = [ "deno_error", "raw-window-handle", "serde", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "wgpu-core", "wgpu-types", @@ -4559,7 +4559,7 @@ dependencies = [ "once_cell", "rustls-tokio-stream", "serde", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", ] @@ -4573,7 +4573,7 @@ dependencies = [ "deno_error", "deno_web", "rusqlite", - "thiserror 2.0.17", + "thiserror 2.0.18", ] [[package]] @@ -4621,7 +4621,7 @@ dependencies = [ "rand 0.8.5", "serde", "serde_json", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "tokio-util", "url", @@ -4646,7 +4646,7 @@ dependencies = [ "rand 0.8.5", "rusqlite", "serde_json", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "tokio-stream", "uuid", @@ -5474,14 +5474,13 @@ checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" [[package]] name = "filetime" -version = "0.2.26" +version = "0.2.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc0505cd1b6fa6580283f6bdf70a73fcf4aba1184038c90902b92b3dd0df63ed" +checksum = "f98844151eee8917efc50bd9e8318cb963ae8b297431495d3f758616ea5c57db" dependencies = [ "cfg-if", "libc", "libredox", - "windows-sys 0.60.2", ] [[package]] @@ -6575,7 +6574,7 @@ dependencies = [ "reqwest 0.12.28", "serde", "serde_json", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "ureq", "windows-sys 0.60.2", @@ -6600,7 +6599,7 @@ dependencies = [ "once_cell", "rand 0.9.0", "serde", - "thiserror 2.0.17", + "thiserror 2.0.18", "tinyvec", "tokio", "tracing", @@ -6624,7 +6623,7 @@ dependencies = [ "resolv-conf", "serde", "smallvec", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "tracing", ] @@ -7147,7 +7146,7 @@ dependencies = [ "percent-encoding", "serde", "serde_json", - "thiserror 2.0.17", + "thiserror 2.0.18", "url", ] @@ -7427,7 +7426,7 @@ dependencies = [ "pest_derive", "regex", "serde_json", - "thiserror 2.0.17", + "thiserror 2.0.18", ] [[package]] @@ -7617,7 +7616,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "tokio-util", "tower 0.5.3", @@ -7641,7 +7640,7 @@ dependencies = [ "serde", "serde-value", "serde_json", - "thiserror 2.0.17", + "thiserror 2.0.18", ] [[package]] @@ -7679,7 +7678,7 @@ dependencies = [ "pin-project", "serde", "serde_json", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "tokio-util", "tracing", @@ -8470,7 +8469,7 @@ dependencies = [ "quote", "syn 2.0.114", "termcolor", - "thiserror 2.0.17", + "thiserror 2.0.18", ] [[package]] @@ -8495,7 +8494,7 @@ dependencies = [ "serde", "serde_json", "socket2 0.5.10", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "tokio-native-tls", "tokio-util", @@ -8527,7 +8526,7 @@ dependencies = [ "serde_json", "sha1", "sha2 0.10.9", - "thiserror 2.0.17", + "thiserror 2.0.18", "uuid", ] @@ -8697,7 +8696,7 @@ dependencies = [ "serde", "serde_json", "sys_traits", - "thiserror 2.0.17", + "thiserror 2.0.18", "url", ] @@ -8845,7 +8844,7 @@ dependencies = [ "num-format", "serde", "serde_json", - "thiserror 2.0.17", + "thiserror 2.0.18", "typetag", "windows-sys 0.48.0", ] @@ -9120,7 +9119,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "tracing", "url", @@ -10202,7 +10201,7 @@ dependencies = [ "lazy_static", "memchr", "parking_lot", - "thiserror 2.0.17", + "thiserror 2.0.18", ] [[package]] @@ -10387,7 +10386,7 @@ dependencies = [ "rustc-hash 2.1.1", "rustls 0.23.35", "socket2 0.6.1", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "tracing", "web-time", @@ -10409,7 +10408,7 @@ dependencies = [ "rustls 0.23.35", "rustls-pki-types", "slab", - "thiserror 2.0.17", + "thiserror 2.0.18", "tinyvec", "tracing", "web-time", @@ -10696,7 +10695,7 @@ checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" dependencies = [ "getrandom 0.2.17", "libredox", - "thiserror 2.0.17", + "thiserror 2.0.18", ] [[package]] @@ -10873,7 +10872,7 @@ dependencies = [ "http 1.4.0", "reqwest 0.13.1", "serde", - "thiserror 2.0.17", + "thiserror 2.0.18", "tower-service", ] @@ -10892,7 +10891,7 @@ dependencies = [ "reqwest 0.13.1", "reqwest-middleware", "retry-policies", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "tracing", "wasmtimer", @@ -11015,7 +11014,7 @@ dependencies = [ "serde", "serde_json", "sse-stream", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "tokio-stream", "tokio-util", @@ -11938,7 +11937,7 @@ dependencies = [ "num-bigint", "serde", "smallvec", - "thiserror 2.0.17", + "thiserror 2.0.18", "v8", ] @@ -12167,7 +12166,7 @@ checksum = "297f631f50729c8c99b84667867963997ec0b50f32b2a7dbcab828ef0541e8bb" dependencies = [ "num-bigint", "num-traits", - "thiserror 2.0.17", + "thiserror 2.0.18", "time", ] @@ -12460,7 +12459,7 @@ dependencies = [ "serde_json", "sha2 0.10.9", "smallvec", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "tokio-stream", "tracing", @@ -12546,7 +12545,7 @@ dependencies = [ "smallvec", "sqlx-core", "stringprep", - "thiserror 2.0.17", + "thiserror 2.0.18", "tracing", "uuid", "whoami", @@ -12587,7 +12586,7 @@ dependencies = [ "smallvec", "sqlx-core", "stringprep", - "thiserror 2.0.17", + "thiserror 2.0.18", "tracing", "uuid", "whoami", @@ -12613,7 +12612,7 @@ dependencies = [ "serde", "serde_urlencoded", "sqlx-core", - "thiserror 2.0.17", + "thiserror 2.0.18", "tracing", "url", "uuid", @@ -13345,7 +13344,7 @@ dependencies = [ "tantivy-stacker", "tantivy-tokenizer-api", "tempfile", - "thiserror 2.0.17", + "thiserror 2.0.18", "time", "uuid", "winapi", @@ -13517,11 +13516,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.17" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" dependencies = [ - "thiserror-impl 2.0.17", + "thiserror-impl 2.0.18", ] [[package]] @@ -13537,9 +13536,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.17" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", @@ -14191,7 +14190,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "786d480bce6247ab75f005b14ae1624ad978d3029d9113f0a22fa1ac773faeaf" dependencies = [ "crossbeam-channel", - "thiserror 2.0.17", + "thiserror 2.0.18", "time", "tracing-subscriber", ] @@ -15013,7 +15012,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2eeee3bdea6257cc36d756fa745a70f9d393571e47d69e0ed97581676a5369ca" dependencies = [ "deno_error", - "thiserror 2.0.17", + "thiserror 2.0.18", ] [[package]] @@ -15243,7 +15242,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill" -version = "1.610.1" +version = "1.611.0" dependencies = [ "anyhow", "aws-sdk-config", @@ -15306,7 +15305,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.610.1" +version = "1.611.0" dependencies = [ "anyhow", "argon2", @@ -15393,7 +15392,7 @@ dependencies = [ "sqlx", "tar", "tempfile", - "thiserror 2.0.17", + "thiserror 2.0.18", "time", "tinyvector", "tokenizers", @@ -15431,7 +15430,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.610.1" +version = "1.611.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -15441,7 +15440,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.610.1" +version = "1.611.0" dependencies = [ "chrono", "lazy_static", @@ -15455,7 +15454,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.610.1" +version = "1.611.0" dependencies = [ "anyhow", "axum 0.7.9", @@ -15464,7 +15463,7 @@ dependencies = [ "serde", "serde_json", "sqlx", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "tracing", "uuid", @@ -15474,7 +15473,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.610.1" +version = "1.611.0" dependencies = [ "anyhow", "async-recursion", @@ -15545,7 +15544,7 @@ dependencies = [ "systemstat", "tar", "tempfile", - "thiserror 2.0.17", + "thiserror 2.0.18", "tikv-jemalloc-ctl", "tokio", "tokio-postgres 0.7.13", @@ -15568,7 +15567,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.610.1" +version = "1.611.0" dependencies = [ "regex", "serde", @@ -15583,7 +15582,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.610.1" +version = "1.611.0" dependencies = [ "anyhow", "astral-tokio-tar", @@ -15607,7 +15606,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.610.1" +version = "1.611.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -15623,7 +15622,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.610.1" +version = "1.611.0" dependencies = [ "anyhow", "async-trait", @@ -15643,7 +15642,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.610.1" +version = "1.611.0" dependencies = [ "anyhow", "async-oauth2", @@ -15667,7 +15666,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.610.1" +version = "1.611.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -15676,7 +15675,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.610.1" +version = "1.611.0" dependencies = [ "anyhow", "lazy_static", @@ -15688,7 +15687,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.610.1" +version = "1.611.0" dependencies = [ "anyhow", "serde_json", @@ -15700,7 +15699,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.610.1" +version = "1.611.0" dependencies = [ "anyhow", "gosyn", @@ -15712,7 +15711,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.610.1" +version = "1.611.0" dependencies = [ "anyhow", "lazy_static", @@ -15724,7 +15723,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.610.1" +version = "1.611.0" dependencies = [ "anyhow", "serde_json", @@ -15736,7 +15735,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.610.1" +version = "1.611.0" dependencies = [ "anyhow", "nu-parser", @@ -15747,7 +15746,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.610.1" +version = "1.611.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15758,7 +15757,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.610.1" +version = "1.611.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15771,7 +15770,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.610.1" +version = "1.611.0" dependencies = [ "anyhow", "async-recursion", @@ -15795,7 +15794,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.610.1" +version = "1.611.0" dependencies = [ "anyhow", "lazy_static", @@ -15809,7 +15808,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.610.1" +version = "1.611.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -15826,7 +15825,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.610.1" +version = "1.611.0" dependencies = [ "anyhow", "lazy_static", @@ -15840,7 +15839,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.610.1" +version = "1.611.0" dependencies = [ "anyhow", "lazy_static", @@ -15859,7 +15858,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.610.1" +version = "1.611.0" dependencies = [ "anyhow", "serde", @@ -15870,7 +15869,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.610.1" +version = "1.611.0" dependencies = [ "anyhow", "async-recursion", @@ -15896,7 +15895,7 @@ dependencies = [ "serde_urlencoded", "sql-builder", "sqlx", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "tracing", "ulid", @@ -15907,7 +15906,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.610.1" +version = "1.611.0" dependencies = [ "wasm-bindgen", "wasm-bindgen-test", @@ -15917,7 +15916,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.610.1" +version = "1.611.0" dependencies = [ "anyhow", "async-once-cell", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 28126fe054..e940b0b625 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.610.1" +version = "1.611.0" authors.workspace = true edition.workspace = true @@ -35,7 +35,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal"] [workspace.package] -version = "1.610.1" +version = "1.611.0" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 5f635d38a9..d20e4ff7d2 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.610.1 + version: 1.611.0 title: Windmill API contact: diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index d391e243d4..07cd3c0364 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.610.1"; +export const VERSION = "v1.611.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/src/main.ts b/cli/src/main.ts index 867d1fa7fb..d1797ecd28 100644 --- a/cli/src/main.ts +++ b/cli/src/main.ts @@ -70,7 +70,7 @@ export { // } // }); -export const VERSION = "1.610.1"; +export const VERSION = "1.611.0"; // Re-exported from constants.ts to maintain backwards compatibility export { WM_FORK_PREFIX } from "./core/constants.ts"; diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 13c9d67dde..c60dade046 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.610.1", + "version": "1.611.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.610.1", + "version": "1.611.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { @@ -835,6 +835,7 @@ "version": "1.7.1", "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.7.1.tgz", "integrity": "sha512-o1uhUASyo921r2XtHYOHy7gdkGLge8ghBEQHMWmyJFoXlpU58kIrhhN3w26lpQb6dspetweapMn2CSNwQ8I4wg==", + "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -846,6 +847,7 @@ "version": "1.7.1", "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.7.1.tgz", "integrity": "sha512-PVtJr5CmLwYAU9PZDMITZoR5iAOShYREoR45EyyLrbntV50mdePTgUn4AmOw90Ifcj+x2kRjdzr1HP3RrNiHGA==", + "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -856,6 +858,7 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.1.0.tgz", "integrity": "sha512-WI0DdZ8xFSbgMjR1sFsKABJ/C5OnRrjT06JXbZKexJGrDuPTzZdDYfFlsgcCXCyf+suG5QU2e/y1Wo2V/OapLQ==", + "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -1345,6 +1348,7 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.0.tgz", "integrity": "sha512-Fq6DJW+Bb5jaWE69/qOE0D1TUN9+6uWhCeZpdnSBk14pjLcCWR7Q8n49PTSPHazM37JqrsdpEthXy2xn6jWWiA==", + "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -1546,6 +1550,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1562,6 +1567,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1578,6 +1584,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1594,6 +1601,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1610,6 +1618,7 @@ "cpu": [ "arm" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1626,6 +1635,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1642,6 +1652,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1658,6 +1669,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1674,6 +1686,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1690,6 +1703,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1706,6 +1720,7 @@ "cpu": [ "wasm32" ], + "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -1722,6 +1737,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1738,6 +1754,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -2358,6 +2375,7 @@ "version": "0.10.1", "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.1.tgz", "integrity": "sha512-9tTaPJLSiejZKx+Bmog4uSubteqTvFrVrURwkmHixBo0G4seD0zUxp98E1DzUBJxLQ3NPwXrGKDiVjwx/DpPsg==", + "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -7196,7 +7214,7 @@ "version": "1.21.7", "resolved": "https://registry.npmjs.org/jiti/-/jiti-1.21.7.tgz", "integrity": "sha512-/imKNG4EbWNrVjoNC/1H5/9GFy+tqjGBHCaSsN+P2RnPqjsLmv6UD3Ej+Kj8nBWaRAwyk7kK5ZUc+OEatnTR3A==", - "devOptional": true, + "dev": true, "license": "MIT", "bin": { "jiti": "bin/jiti.js" @@ -7632,6 +7650,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7652,6 +7671,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7672,6 +7692,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7692,6 +7713,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7712,6 +7734,7 @@ "cpu": [ "arm" ], + "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7732,6 +7755,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7752,6 +7776,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7772,6 +7797,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7792,6 +7818,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7812,6 +7839,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7832,6 +7860,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -12469,6 +12498,21 @@ } } }, + "node_modules/svelte-check/node_modules/picomatch": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", + "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, "node_modules/svelte-eslint-parser": { "version": "0.43.0", "resolved": "https://registry.npmjs.org/svelte-eslint-parser/-/svelte-eslint-parser-0.43.0.tgz", diff --git a/frontend/package.json b/frontend/package.json index 3df060d6c3..33d7756634 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.610.1", + "version": "1.611.0", "scripts": { "dev": "vite dev", "build": "vite build", diff --git a/lsp/Pipfile b/lsp/Pipfile index bca80202ad..d5ef2c2c1f 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,8 +4,8 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.610.1" -wmill_pg = ">=1.610.1" +wmill = ">=1.611.0" +wmill_pg = ">=1.611.0" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index 24e34c8f3a..1b6e2dcb67 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: '3.0.3' info: - version: 1.610.1 + version: 1.611.0 title: OpenFlow Spec contact: name: Ruben Fiszel diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index 57eae4a29b..234c5dfee9 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. - ModuleVersion = '1.610.1' + ModuleVersion = '1.611.0' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index 09b0d2beec..a24874f024 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.610.1" +version = "1.611.0" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/python-client/wmill_pg/pyproject.toml b/python-client/wmill_pg/pyproject.toml index c4d6b47326..5889857365 100644 --- a/python-client/wmill_pg/pyproject.toml +++ b/python-client/wmill_pg/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill-pg" -version = "1.610.1" +version = "1.611.0" description = "An extension client for the wmill client library focused on pg" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index 3c9618a769..12104e55c4 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.610.1", + "version": "1.611.0", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./sqlUtils.ts", "./client.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index 2c65914f87..6043d7a468 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.610.1", + "version": "1.611.0", "author": "Ruben Fiszel", "license": "Apache 2.0", "devDependencies": { diff --git a/version.txt b/version.txt index 3173f5e2c6..cc23b145bf 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.610.1 +1.611.0 From c04eb371ccd805e8d0d0a03b4ef654c7a8131ccd Mon Sep 17 00:00:00 2001 From: Pyra <92104930+pyranota@users.noreply.github.com> Date: Mon, 19 Jan 2026 16:48:50 +0100 Subject: [PATCH 18/19] feat: move job metrics from ee to ce (#7608) * open source job metrics Signed-off-by: pyranota * fix Signed-off-by: pyranota * update Signed-off-by: pyranota --------- Signed-off-by: pyranota --- backend/windmill-worker/src/handle_child.rs | 47 ++++++++----------- .../components/MemoryFootprintViewer.svelte | 8 +--- 2 files changed, 21 insertions(+), 34 deletions(-) diff --git a/backend/windmill-worker/src/handle_child.rs b/backend/windmill-worker/src/handle_child.rs index 18e053116a..8de0899c04 100644 --- a/backend/windmill-worker/src/handle_child.rs +++ b/backend/windmill-worker/src/handle_child.rs @@ -36,7 +36,6 @@ use std::{io, panic, time::Duration}; use tracing::{trace_span, Instrument}; use uuid::Uuid; -#[cfg(feature = "enterprise")] use windmill_common::job_metrics; #[cfg(target_os = "linux")] @@ -696,7 +695,6 @@ where let mut i = 0; - #[cfg(feature = "enterprise")] let mut memory_metric_id: Result = Err(Error::NotFound("not yet initialized".to_string())); @@ -731,32 +729,26 @@ where let update_job_row = i == 2 || (!*SLOW_LOGS && (i < 20 || (i < 120 && i % 5 == 0) || i % 10 == 0)) || i % 20 == 0; - if update_job_row { - #[cfg(feature = "enterprise")] - { - if job_id != Uuid::nil() { - if let Connection::Sql(ref db) = conn { - // tracking metric starting at i >= 2 b/c first point it useless and we don't want to track metric for super fast jobs - if i == 2 { - memory_metric_id = job_metrics::register_metric_for_job( - &db, - w_id.to_string(), - job_id, - "memory_kb".to_string(), - job_metrics::MetricKind::TimeseriesInt, - Some("Job Memory Footprint (kB)".to_string()), - ) - .await; - } - if let Ok(ref metric_id) = memory_metric_id { - if let Err(err) = job_metrics::record_metric(&db, w_id.to_string(), job_id, metric_id.to_owned(), job_metrics::MetricNumericValue::Integer(current_mem)).await { - tracing::error!("Unable to save memory stat for job {} in workspace {}. Error was: {:?}", job_id, w_id, err); - } + if update_job_row && job_id != Uuid::nil() { + if let Connection::Sql(ref db) = conn { + // tracking metric starting at i >= 2 b/c first point it useless and we don't want to track metric for super fast jobs + if i == 2 { + memory_metric_id = job_metrics::register_metric_for_job( + &db, + w_id.to_string(), + job_id, + "memory_kb".to_string(), + job_metrics::MetricKind::TimeseriesInt, + Some("Job Memory Footprint (kB)".to_string()), + ) + .await; + } + if let Ok(ref metric_id) = memory_metric_id { + if let Err(err) = job_metrics::record_metric(&db, w_id.to_string(), job_id, metric_id.to_owned(), job_metrics::MetricNumericValue::Integer(current_mem)).await { + tracing::error!("Unable to save memory stat for job {} in workspace {}. Error was: {:?}", job_id, w_id, err); } } } - } - if job_id != Uuid::nil() { if matches!(conn, Connection::Http(_)) { if i % 4 != 0 { // only ping every 4th time (2s) on http agent mode @@ -764,8 +756,8 @@ where } } let ping_job_status = ping_job_status(&conn, &job_id, Some(*mem_peak), if current_mem > 0 { Some(current_mem) } else { None }).await.unwrap_or_else(|e| { - tracing::error!("Unable to ping job status for job {job_id}. Error was: {:?}", e); - PingJobStatusResponse { + tracing::error!("Unable to ping job status for job {job_id}. Error was: {:?}", e); + PingJobStatusResponse { canceled_by: None, canceled_reason: None, already_completed: false, @@ -782,7 +774,6 @@ where break } } - } }, ); } diff --git a/frontend/src/lib/components/MemoryFootprintViewer.svelte b/frontend/src/lib/components/MemoryFootprintViewer.svelte index afcdfb82a4..66da99e6e6 100644 --- a/frontend/src/lib/components/MemoryFootprintViewer.svelte +++ b/frontend/src/lib/components/MemoryFootprintViewer.svelte @@ -1,7 +1,7 @@
- {#if !$enterpriseLicense} - - Job metrics are only available on Windmill Enterprise Edition. - - {:else if (jobMemoryStats?.length ?? 0) === 0} + {#if (jobMemoryStats?.length ?? 0) === 0} No data points available for this job. Metrics are recorded only for jobs running for more than 500ms. Date: Mon, 19 Jan 2026 17:10:59 +0000 Subject: [PATCH 19/19] feat(cli): add folders as branch-specific items Folders can now be configured as branch-specific items in wmill.yaml: ```yaml gitBranches: staging: specificItems: folders: - "f/env_*" - "u/admin/config" ``` This allows different folder configurations per branch. A folder like `f/env_staging/folder.meta.yaml` becomes `f/env_staging.main/folder.meta.yaml` on the main branch. Changes: - Add `folders?: string[]` to SpecificItemsConfig - Add folder detection in isSpecificItem() - Add folder pattern to toBranchSpecificPath/fromBranchSpecificPath - Add folder pattern to isBranchSpecificFile/isCurrentBranchFile - Add 8 new tests for folder functionality Co-Authored-By: Claude Opus 4.5 --- cli/src/core/specific_items.ts | 39 ++++++++++++++++++++++-- cli/test/specific_items.test.ts | 53 +++++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+), 3 deletions(-) diff --git a/cli/src/core/specific_items.ts b/cli/src/core/specific_items.ts index bef871b598..a81652c4db 100644 --- a/cli/src/core/specific_items.ts +++ b/cli/src/core/specific_items.ts @@ -8,6 +8,7 @@ export interface SpecificItemsConfig { variables?: string[]; resources?: string[]; triggers?: string[]; + folders?: string[]; } // Define all branch-specific file types (computed lazily) @@ -15,6 +16,7 @@ function getBranchSpecificTypes() { return { variable: '.variable.yaml', resource: '.resource.yaml', + folder: '/folder.meta.yaml', // Generate trigger patterns from the list ...Object.fromEntries( TRIGGER_TYPES.map(t => [`${t}_trigger`, `.${t}_trigger.yaml`]) @@ -142,6 +144,16 @@ export function isSpecificItem(path: string, specificItems: SpecificItemsConfig return specificItems.triggers ? matchesPatterns(path, specificItems.triggers) : false; } + // Check for folder meta files + if (path.endsWith('/folder.meta.yaml')) { + if (specificItems.folders) { + // Match against the folder path (without /folder.meta.yaml) + const folderPath = path.slice(0, -'/folder.meta.yaml'.length); + return matchesPatterns(folderPath, specificItems.folders); + } + return false; + } + // Check for resource files using the standard detection function if (isFileResource(path)) { // Extract the base path without the file extension to match against patterns @@ -197,7 +209,20 @@ export function fromBranchSpecificPath(branchSpecificPath: string, branchName: s const sanitizedBranchName = branchName.replace(/[\/\\:*?"<>|.]/g, '_'); const escapedBranchName = sanitizedBranchName.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); - // Check for resource file pattern first + // Check for folder pattern: path.branchName/folder.meta.yaml -> path/folder.meta.yaml + const folderPattern = new RegExp(`\\.${escapedBranchName}(/folder\\.meta\\.yaml)$`); + const folderMatch = branchSpecificPath.match(folderPattern); + + if (folderMatch) { + const extension = folderMatch[1]; + const pathWithoutBranchAndExtension = branchSpecificPath.substring( + 0, + branchSpecificPath.length - `.${sanitizedBranchName}${extension}`.length + ); + return `${pathWithoutBranchAndExtension}${extension}`; + } + + // Check for resource file pattern const resourceFilePattern = new RegExp(`\\.${escapedBranchName}(\\.resource\\.file\\..+)$`); const resourceFileMatch = branchSpecificPath.match(resourceFilePattern); @@ -283,7 +308,11 @@ export function isCurrentBranchFile(path: string, branchOverride?: string): bool // Use cached pattern or create and cache new one let pattern = branchPatternCache.get(currentBranch); if (!pattern) { - pattern = new RegExp(`\\.${escapedBranchName}\\.${buildYamlTypePattern()}\\.yaml$|\\.${escapedBranchName}\\.resource\\.file\\..+$`); + pattern = new RegExp( + `\\.${escapedBranchName}\\.${buildYamlTypePattern()}\\.yaml$|` + + `\\.${escapedBranchName}\\.resource\\.file\\..+$|` + + `\\.${escapedBranchName}/folder\\.meta\\.yaml$` + ); branchPatternCache.set(currentBranch, pattern); } @@ -296,5 +325,9 @@ export function isCurrentBranchFile(path: string, branchOverride?: string): bool */ export function isBranchSpecificFile(path: string): boolean { const yamlTypePattern = buildYamlTypePattern(); - return new RegExp(`\\.[^.]+\\.${yamlTypePattern}\\.yaml$|\\.[^.]+\\.resource\\.file\\..+$`).test(path); + return new RegExp( + `\\.[^.]+\\.${yamlTypePattern}\\.yaml$|` + + `\\.[^.]+\\.resource\\.file\\..+$|` + + `\\.[^.]+/folder\\.meta\\.yaml$` + ).test(path); } diff --git a/cli/test/specific_items.test.ts b/cli/test/specific_items.test.ts index 4448f32db7..a638540657 100644 --- a/cli/test/specific_items.test.ts +++ b/cli/test/specific_items.test.ts @@ -336,3 +336,56 @@ Deno.test("branchOverride: getSpecificItemsForCurrentBranch merges common and br assertEquals(result?.resources, ["shared/**"]); assertEquals(result?.triggers, ["dev/triggers/**"]); }); + +// ============================================================================= +// FOLDER BRANCH-SPECIFIC TESTS +// ============================================================================= + +Deno.test("toBranchSpecificPath: converts folder path to branch-specific", () => { + const result = toBranchSpecificPath("f/my_folder/folder.meta.yaml", "main"); + assertEquals(result, "f/my_folder.main/folder.meta.yaml"); +}); + +Deno.test("toBranchSpecificPath: converts nested folder path to branch-specific", () => { + const result = toBranchSpecificPath("f/parent/child/folder.meta.yaml", "develop"); + assertEquals(result, "f/parent/child.develop/folder.meta.yaml"); +}); + +Deno.test("fromBranchSpecificPath: converts branch-specific folder back to base", () => { + const result = fromBranchSpecificPath("f/my_folder.main/folder.meta.yaml", "main"); + assertEquals(result, "f/my_folder/folder.meta.yaml"); +}); + +Deno.test("fromBranchSpecificPath: handles sanitized branch names for folders", () => { + const result = fromBranchSpecificPath("f/my_folder.feature_test/folder.meta.yaml", "feature/test"); + assertEquals(result, "f/my_folder/folder.meta.yaml"); +}); + +Deno.test("isSpecificItem: matches folder paths with glob pattern", () => { + const config: SpecificItemsConfig = { + folders: ["f/env_*"], + }; + assertEquals(isSpecificItem("f/env_staging/folder.meta.yaml", config), true); + assertEquals(isSpecificItem("f/env_production/folder.meta.yaml", config), true); + assertEquals(isSpecificItem("f/other/folder.meta.yaml", config), false); +}); + +Deno.test("isBranchSpecificFile: detects branch-specific folder files", () => { + assertEquals(isBranchSpecificFile("f/my_folder.main/folder.meta.yaml"), true); + assertEquals(isBranchSpecificFile("f/my_folder.develop/folder.meta.yaml"), true); + assertEquals(isBranchSpecificFile("f/my_folder/folder.meta.yaml"), false); +}); + +Deno.test("isCurrentBranchFile: detects branch-specific folder for current branch", () => { + assertEquals(isCurrentBranchFile("f/my_folder.staging/folder.meta.yaml", "staging"), true); + assertEquals(isCurrentBranchFile("f/my_folder.staging/folder.meta.yaml", "production"), false); + assertEquals(isCurrentBranchFile("f/my_folder/folder.meta.yaml", "staging"), false); +}); + +Deno.test("round-trip: folder path conversion", () => { + const original = "f/configs/env_folder/folder.meta.yaml"; + const branch = "feature/new-env"; + const branchSpecific = toBranchSpecificPath(original, branch); + const restored = fromBranchSpecificPath(branchSpecific, branch); + assertEquals(restored, original); +});