diff --git a/.claude/hooks/allow-fileops-in-tmp.sh b/.claude/hooks/allow-fileops-in-tmp.sh index ce2ce81831..665c23cf0a 100755 --- a/.claude/hooks/allow-fileops-in-tmp.sh +++ b/.claude/hooks/allow-fileops-in-tmp.sh @@ -2,10 +2,12 @@ # PreToolUse allowance for scratch file ops: auto-allow a single, plain, single-line # `mkdir` / `cp` / `mv` / `touch` / `chmod` / `tar` / `unzip` whose every path operand # resolves under /tmp. Anything else makes no decision (exit 0) and falls back to the normal -# permission flow — where `Bash(mv:*)` and `Bash(chmod:*)` in the `ask` list prompt. A -# PreToolUse `allow` overrides those ask rules, which is why this is a hook and not an allow -# rule: permission rules match a command prefix, so they can only constrain the FIRST operand. -# `cp /tmp/x ~/.zshrc` matches a `cp /tmp/` prefix, and requiring every operand is the point. +# permission flow, except for `mv` and `chmod`: those get an explicit `ask`, the only prompt +# they get (see lib-guarded-verb.sh). +# +# This is a hook rather than an allow rule because permission rules match a command prefix, so +# they can only constrain the FIRST operand. `cp /tmp/x ~/.zshrc` matches a `cp /tmp/` prefix, +# and requiring every operand is the point. # # Requiring the sources under /tmp too (not just the destination) keeps this from becoming a # read-exfiltration path around the `Read(**/.env)` / `Read(**/secrets/**)` deny rules: a copy @@ -31,6 +33,7 @@ # # Assumes GNU `realpath` (-m) and `jq`, both present in this repo's Linux dev env. set -uo pipefail +. "${BASH_SOURCE[0]%/*}/lib-guarded-verb.sh" input=$(cat) command -v jq >/dev/null 2>&1 || exit 0 @@ -38,8 +41,19 @@ cmd=$(printf '%s' "$input" | jq -r '.tool_input.command // empty' 2>/dev/null) [ -z "$cmd" ] && exit 0 cwd=$(printf '%s' "$input" | jq -r '.cwd // empty' 2>/dev/null) +# Every bail-out below goes through `defer`: `mv` and `chmod` prompt from here, since no rule +# covers them, while the other verbs stay silent and leave the decision to the normal flow. +guarded=0 +for verb in mv chmod; do + runs_verb "$verb" "$cmd" && { guarded=1; break; } +done +defer() { + [ "$guarded" = 1 ] && decide ask "$1" + exit 0 +} + # A newline separates commands, and the tokenizer below only reads the first line — defer. -case "$cmd" in *$'\n'*) exit 0 ;; esac +case "$cmd" in *$'\n'*) defer "multi-line command" ;; esac read -r -a toks <<< "$cmd" @@ -65,11 +79,6 @@ under_tmp() { return 1 } -allow() { - jq -nc --arg r "$1" '{hookSpecificOutput:{hookEventName:"PreToolUse",permissionDecision:"allow",permissionDecisionReason:$r}}' - exit 0 -} - # Bare command word only; wrappers (`timeout cp`), env prefixes, and `/bin/cp` defer. # Options are an allowlist per command, so anything that changes how symlinks are followed # defers instead of needing enumeration. `cp -L` / `-H` matter most: they dereference while @@ -84,7 +93,7 @@ case "${toks[0]:-}" in chmod) takes_mode=1; ok_opts='Rvfc' ;; # chmod's first operand is a mode, not a path tar) ok_flags='xctzjJavfC'; val_flags='fC' ;; unzip) ok_flags='oqnljvd'; val_flags='d' ;; - *) exit 0 ;; + *) defer "not the leading command word" ;; esac # ---------------------------------------------------------------- tar / unzip @@ -101,18 +110,18 @@ if [ -n "${ok_flags:-}" ]; then flags="${t#-}" # Allowlist: a long option, -P/--absolute-names, --transform, -I and friends all # leave a residue here and defer rather than being enumerated as denials. - [ -n "$(printf '%s' "$flags" | tr -d "$ok_flags")" ] && exit 0 + [ -n "$(printf '%s' "$flags" | tr -d "$ok_flags")" ] && defer "unrecognized option \`$t\`" case "$flags" in *x*) extracting=1 ;; esac case "${toks[0]}$flags" in unzip*[lv]*) listing=1 ;; esac # A flag consuming the next token must be alone in its bundle's final position # (`-xzf a.tar`), else the token it eats is ambiguous. - case "${flags%?}" in *[$val_flags]*) exit 0 ;; esac + case "${flags%?}" in *[$val_flags]*) defer "ambiguous option bundle \`$t\`" ;; esac case "${flags: -1}" in [$val_flags]) val="${toks[$i]:-}" i=$((i + 1)) - [ -n "$val" ] || exit 0 - under_tmp "$val" || exit 0 + [ -n "$val" ] || defer "option \`$t\` has no value" + under_tmp "$val" || defer "\`$val\` is outside /tmp" case "${flags: -1}" in f) saw_archive=1 ;; C | d) saw_dest=1 ;; @@ -126,17 +135,18 @@ if [ -n "${ok_flags:-}" ]; then # Positional. For tar these are sources (create) or member names (extract); for unzip the # first is the archive. Requiring every one under /tmp is conservative for member names, # which are not filesystem paths — those defer rather than being wrongly allowed. - under_tmp "$t" || exit 0 + under_tmp "$t" || defer "\`$t\` is outside /tmp" [ "${toks[0]}" = "unzip" ] && saw_archive=1 done - [ "$saw_archive" = 1 ] || exit 0 # tar without -f reads a tape/stdin; unzip needs an archive + # tar without -f reads a tape/stdin; unzip needs an archive + [ "$saw_archive" = 1 ] || defer "no archive operand" # Writes land relative to the working directory unless a destination was given. `unzip -l` # and `-v` only list, so they need no destination. if [ "$extracting" = 1 ] || { [ "${toks[0]}" = "unzip" ] && [ "$listing" = 0 ]; }; then - [ "$saw_dest" = 1 ] || under_tmp "${cwd:-$PWD}" || exit 0 + [ "$saw_dest" = 1 ] || under_tmp "${cwd:-$PWD}" || defer "extraction target is outside /tmp" fi - allow "archive paths and extraction target are under /tmp" + decide allow "archive paths and extraction target are under /tmp" fi # ------------------------------------------- mkdir / cp / mv / touch / chmod @@ -155,7 +165,7 @@ while [ "$i" -lt "${#toks[@]}" ]; do case "$t" in -?*) # Allowlist: long options and the dereferencing flags leave a residue and defer. - [ -n "$(printf '%s' "${t#-}" | tr -d "$ok_opts")" ] && exit 0 + [ -n "$(printf '%s' "${t#-}" | tr -d "$ok_opts")" ] && defer "unrecognized option \`$t\`" continue ;; esac @@ -165,15 +175,15 @@ while [ "$i" -lt "${#toks[@]}" ]; do if [ "$takes_mode" = 1 ] && [ "$seen_mode" = 0 ]; then case "$t" in [0-7] | [0-7][0-7] | [0-7][0-7][0-7] | [0-7][0-7][0-7][0-7]) ;; - *) printf '%s' "$t" | grep -Eq '^[ugoa]*[+=-][rwxXst]*(,[ugoa]*[+=-][rwxXst]*)*$' || exit 0 ;; + *) printf '%s' "$t" | grep -Eq '^[ugoa]*[+=-][rwxXst]*(,[ugoa]*[+=-][rwxXst]*)*$' || defer "unrecognized mode \`$t\`" ;; esac seen_mode=1 continue fi - under_tmp "$t" || exit 0 + under_tmp "$t" || defer "\`$t\` is outside /tmp" path_operand=1 done -[ "$path_operand" = 1 ] || exit 0 -allow "every path operand is under /tmp" +[ "$path_operand" = 1 ] || defer "no path operand" +decide allow "every path operand is under /tmp" diff --git a/.claude/hooks/guard-rm-outside-tmp.sh b/.claude/hooks/guard-rm-outside-tmp.sh index 66d4dd27b5..5aff497d89 100755 --- a/.claude/hooks/guard-rm-outside-tmp.sh +++ b/.claude/hooks/guard-rm-outside-tmp.sh @@ -1,9 +1,9 @@ #!/usr/bin/env bash # PreToolUse guard for `rm`: auto-allow ONLY a single, plain, single-line `rm` whose every # operand is a whitelisted target — under /tmp, or inside a git working tree located in $HOME -# (a version-controlled project dir). Anything else makes no decision (exit 0) and falls back -# to the normal permission flow, where the `Bash(rm:*)` ask rule prompts (classifier as a -# backstop). +# (a version-controlled project dir). Any other command that runs `rm` gets an explicit `ask`, +# which is the ordinary permission prompt and the only one `rm` gets (see lib-guarded-verb.sh); +# a command that runs no `rm` at all makes no decision (exit 0). # # The git-tree allowance trades on "this is a project under version control" being lower-stakes # than a delete elsewhere — NOT on full recoverability: committed content is restorable via git, @@ -19,13 +19,14 @@ # # The git-repo allowance covers targets inside a git working tree under $HOME, and the tree's # own root folder only when it is a linked worktree (`.git` is a pointer file, so history in -# the main repo survives); a primary checkout's root (`.git` is a history dir) and any `.git` -# path are never auto-allowed. Globs auto-allow only under /tmp — elsewhere their expansion +# the main repo survives); a primary checkout's root (`.git` is a history dir) and any `.git`, +# `.claude` or `.env` path are never auto-allowed. Globs auto-allow only under /tmp — elsewhere their expansion # could reach `.git` or a dotfile the literal checks never see. Relative operands resolve -# against the command's cwd (from the hook input). A PreToolUse `allow` overrides the ask rule. +# against the command's cwd (from the hook input). # # Assumes GNU `realpath` (-m) and `jq`, both present in this repo's Linux dev env. set -uo pipefail +. "${BASH_SOURCE[0]%/*}/lib-guarded-verb.sh" input=$(cat) command -v jq >/dev/null 2>&1 || exit 0 @@ -33,12 +34,20 @@ cmd=$(printf '%s' "$input" | jq -r '.tool_input.command // empty' 2>/dev/null) [ -z "$cmd" ] && exit 0 cwd=$(printf '%s' "$input" | jq -r '.cwd // empty' 2>/dev/null) +# Every bail-out below goes through `defer`, so the forms this guard refuses to reason about — +# compound, quoted, wrapped — still reach the user as a prompt whenever an `rm` runs among them. +runs_verb rm "$cmd" && guarded=1 || guarded=0 +defer() { + [ "$guarded" = 1 ] && decide ask "$1" + exit 0 +} + # A newline separates commands, and the tokenizer below only reads the first line — defer. -case "$cmd" in *$'\n'*) exit 0 ;; esac +case "$cmd" in *$'\n'*) defer "multi-line command" ;; esac read -r -a toks <<< "$cmd" # Bare leading `rm` only; wrappers (`timeout rm`), env prefixes, and `/bin/rm` defer. -[ "${toks[0]:-}" = "rm" ] || exit 0 +[ "${toks[0]:-}" = "rm" ] || defer "rm is not the leading command word" # 0 (allow) iff the canonical path is an auto-allowable rm target: under /tmp, or strictly # inside a git working tree located under $HOME. The walk stops at $HOME, so a dotfiles repo at @@ -48,7 +57,13 @@ allowed_target() { case "$canon" in /tmp/?*) return 0 ;; esac [ -n "${HOME:-}" ] || return 1 case "$canon" in "$HOME"/?*) ;; *) return 1 ;; esac - case "$canon" in *"/.git" | *"/.git/"*) return 1 ;; esac # protect history, not recoverable + # Never auto-allow: history, and the two kinds of path the "it's under version control" + # premise doesn't hold for — the agent's own guards and settings (deleting them is what + # removes the prompt on everything else), and gitignored `.env` files. + case "$canon" in + *"/.git" | *"/.git/"* | *"/.claude" | *"/.claude/"*) return 1 ;; + *"/.env" | *"/.env."*) return 1 ;; + esac d="$canon" while [ "$d" != "/" ] && [ "$d" != "$HOME" ]; do [ -e "$d/.git" ] && { root="$d"; break; } @@ -73,10 +88,10 @@ while [ "$i" -lt "${#toks[@]}" ]; do i=$((i + 1)) # Whitelist every token (flags included, so an operator hidden in a flag like `-rf;rm` # can't slip past): any character outside the safe set makes it unsafe to reason about. - [ -n "$(printf '%s' "$t" | tr -d 'A-Za-z0-9._/*?[]-')" ] && exit 0 + [ -n "$(printf '%s' "$t" | tr -d 'A-Za-z0-9._/*?[]-')" ] && defer "unsafe characters in \`$t\`" # A glob in an option-looking token (`-[-]`) can expand to `--` and turn a later `-name` # into an operand — never a real option, so defer. - case "$t" in -*[*?[]*) exit 0 ;; esac + case "$t" in -*[*?[]*) defer "glob inside the option \`$t\`" ;; esac if [ "$end_opts" = 0 ]; then [ "$t" = "--" ] && { end_opts=1; continue; } # Skip real options only before the first operand. A bare `-` is a filename, and under @@ -89,18 +104,18 @@ while [ "$i" -lt "${#toks[@]}" ]; do had_operand=1 # No wildcard in a non-final path segment (`a/*/b`): it can expand through a symlink # realpath can't see. A slashless glob (`*.rs`) is a final-segment match — fine. - case "$t" in */*) case "${t%/*}" in *[*?[]*) exit 0 ;; esac ;; esac + case "$t" in */*) case "${t%/*}" in *[*?[]*) defer "glob in a non-final segment of \`$t\`" ;; esac ;; esac case "$t" in /*) canon=$(realpath -m -- "$t" 2>/dev/null) ;; *) canon=$(realpath -m -- "${cwd:-$PWD}/$t" 2>/dev/null) ;; esac - [ -n "$canon" ] || exit 0 + [ -n "$canon" ] || defer "cannot resolve \`$t\`" # A glob may auto-allow only under /tmp, where everything is deletable. Elsewhere its # expansion could match `.git`, a dotfile like `.*`, or a nested checkout root that the # literal-path checks never see — so require literal operands in git repos. - case "$t" in *[*?[]*) case "$canon" in /tmp/?*) ;; *) exit 0 ;; esac ;; esac - allowed_target "$canon" || exit 0 + case "$t" in *[*?[]*) case "$canon" in /tmp/?*) ;; *) defer "glob \`$t\` is outside /tmp" ;; esac ;; esac + allowed_target "$canon" || defer "\`$canon\` is outside /tmp and not inside a git checkout in \$HOME" done -[ "$had_operand" = 1 ] || exit 0 -jq -nc '{hookSpecificOutput:{hookEventName:"PreToolUse",permissionDecision:"allow",permissionDecisionReason:"rm operands are under /tmp or inside a git checkout in $HOME"}}' +[ "$had_operand" = 1 ] || defer "no operand" +decide allow 'rm operands are under /tmp or inside a git checkout in $HOME' diff --git a/.claude/hooks/lib-guarded-verb.sh b/.claude/hooks/lib-guarded-verb.sh new file mode 100644 index 0000000000..c4d5f5ac9a --- /dev/null +++ b/.claude/hooks/lib-guarded-verb.sh @@ -0,0 +1,155 @@ +#!/usr/bin/env bash +# Sourced by the PreToolUse guards; not a hook itself. +# +# A permission rule beats a hook: an `ask` rule prompts whatever a PreToolUse hook returns, which +# makes the hook's `allow` dead weight. So settings.json carries no `ask` rule for `rm`, `mv` or +# `chmod`, and the guards own both halves — `allow` what they can prove safe, `ask` for the rest. +# Removing a guard's `ask` path therefore removes that verb's prompt entirely. +# +# `set -f` is global to the sourcing script so that the unquoted word split in runs_verb cannot +# expand a glob operand against the filesystem. Neither guard relies on pathname expansion. +set -f + +# 0 iff ($1) runs as a command word anywhere in ($2). Mirrors how a Bash +# permission rule matches, so that owning the prompt here doesn't narrow what used to prompt: +# the command splits on `; & |` and newlines, and a leading env assignment or process wrapper +# (`timeout 5 rm`, `xargs rm`) is skipped before the command word is read. +# +# The split set also carries the characters that open a nested command — `$(`, backticks and +# `( )` — because a rule matches the verb inside one (`echo $(rm -rf ~)` prompts), and a +# separator that only ends statements would read that as an `echo`. Braces are handled as +# words rather than separators, since splitting on them cuts `xargs -I {} … rm` in half and +# strands the `rm` in a segment that no longer knows a wrapper preceded it. + +# 0 iff ($1) starts with a command that only reads its input. An allowlist, because the +# opposite — naming the shells to avoid — would have to be complete: an unlisted one (`ash`, +# `rbash`, `busybox sh`) executes the body while the guard calls it data. Unrecognized here only +# costs a prompt. Text with no command word in it is not evidence of a reader either. +reads_only() { + local w + for w in $1; do + w="${w//[\"\'\\]/}" + w="${w%%<<*}" # a redirect needs no space: `cat<'* | '<'*) continue ;; esac + case "${w##*/}" in + cat | tee | head | tail | grep | sed | awk | sort | uniq | wc | cut | diff | tr \ + | jq | yq | gh | git | base64 | column | envsubst | python | python3 | node \ + | psql | mysql | sqlite3 | wmill) return 0 ;; + esac + return 1 + done + return 1 +} + +# A heredoc body is data rather than commands only when its delimiter is quoted and nothing +# executes it; a rule doesn't match a verb inside such a body, and a PR body would otherwise +# prompt for every `rm` in its text. Dropping one needs all of that, a delimiter that could +# really open a heredoc, and a terminator line — failing any part, nothing is dropped. +strip_heredoc_bodies() { + local -a lines=() + local line delim rest after trimmed piped quoted i j n + while IFS= read -r line; do lines+=("$line"); done <<< "$1" + n=${#lines[@]} + i=0 + while [ "$i" -lt "$n" ]; do + line="${lines[$i]}" + printf '%s\n' "$line" + i=$((i + 1)) + # A `#` opens a comment, and a comment opens no heredoc — including mid-line, as in + # `echo hi # cat < f`); prose after it means the `<<` sits + # inside a string (`echo "cat < f"` ends its redirect-looking text with the + # closing quote. That also refuses `cat < "f"`, a real heredoc, which only over-prompts. + after="${rest#"$delim"}" + after="${after#"${after%%[![:space:]]*}"}" + case "$after" in + *[\"\'\\]*) continue ;; + "" | '>'* | '<'* | '|'* | [0-9]'>'* | [0-9]'<'*) ;; + *) continue ;; + esac + # A real delimiter is a bare word or one wholly quoted (`<<'EOF'`, `<<\EOF`); a stray quote + # left in it means the `<<` was quoted prose. + quoted=0 + case "$delim" in + \'*\' | \"*\") delim="${delim:1:${#delim}-2}" quoted=1 ;; + \\?*) delim="${delim#\\}" quoted=1 ;; + esac + case "$delim" in + [A-Za-z_]*) ;; + *) continue ;; + esac + case "$delim" in *[!A-Za-z0-9_]*) continue ;; esac + # Only a quoted delimiter makes the body inert. Unquoted, the shell expands it before the + # consumer ever sees it, so a `$(rm -rf ~)` written in the body runs whatever reads it. + [ "$quoted" = 1 ] || continue + # Two commands can see this body: the one the `<<` belongs to, and anything it is then piped + # into. The first is whatever was started last before the `<<`, so splitting the text there + # on separators and substitution openers and taking the final piece finds `cat` in + # `--title "fix(agents): …" --body "$(cat <<`, without the title's parenthesis standing in + # for it. A line continuation (`bash \` then `<<'EOF'`) leaves that piece empty, which is + # not evidence of a reader and so keeps the body. + reads_only "$(printf '%s' "${line%%<<*}" | tr ';&|()`' '\n' | grep -v '^[[:space:]]*$' | tail -1)" || continue + piped="$after" + while :; do + case "$piped" in *'|'*) ;; *) break ;; esac + piped="${piped#*|}" + reads_only "${piped%%|*}" || continue 2 + done + j="$i" + while [ "$j" -lt "$n" ]; do + trimmed="${lines[$j]#"${lines[$j]%%[![:space:]]*}"}" + [ "$trimmed" = "$delim" ] && break + j=$((j + 1)) + done + [ "$j" -lt "$n" ] && i=$((j + 1)) + done +} + +runs_verb() { + local verb="$1" seg w wrapped + while IFS= read -r seg; do + wrapped=0 + for w in $seg; do + # The shell strips quotes and backslashes before it looks up the command, so `'rm'` and + # `r\m` run rm and have to compare equal to it. + w="${w//[\"\'\\]/}" + case "$w" in + "$verb" | */"$verb") return 0 ;; + *=*) ;; # leading env assignment + -* | *'>'* | *'<'*) ;; # a flag, or a leading redirect + [0-9]*) [ "$wrapped" = 1 ] || break ;; # a wrapper's duration, not `1:` in prose + '!' | '{' | '}' | if | then | elif | else | while | until | do) ;; # never the command + timeout | time | nice | nohup | stdbuf | command | builtin | noglob | xargs | sudo | env) + wrapped=1 ;; + # A wrapper's option value is indistinguishable from a command name (`stdbuf -o L rm`), + # so past a wrapper the scan runs to the end of the segment instead of stopping at the + # first ordinary word. Before one, that word is the command and the verb cannot follow + # it. Nothing bounds the scan: a wrapper takes unboundedly many operands + # (`env -u A -u B …`), and any cutoff — a word count, or stopping at the first quoted + # word — drops the prompt for a real `sudo -u 'root' rm`. Prose after a wrapper is the + # price, and it only over-prompts. + *) [ "$wrapped" = 1 ] || break ;; + esac + done + # `tr` and not `${2//[...]}`: a `}` inside the bracket expression closes the expansion + # itself, which silently leaves the command unsplit and every separator unseen. + done <<< "$(strip_heredoc_bodies "$2" | tr ';&|()`' '\n')" + return 1 +} + +# Emit a PreToolUse decision and exit. `ask` is the ordinary permission prompt. +decide() { + jq -nc --arg d "$1" --arg r "$2" \ + '{hookSpecificOutput:{hookEventName:"PreToolUse",permissionDecision:$d,permissionDecisionReason:$r}}' + exit 0 +} diff --git a/.claude/hooks/test-hooks.sh b/.claude/hooks/test-hooks.sh new file mode 100644 index 0000000000..65631b6a48 --- /dev/null +++ b/.claude/hooks/test-hooks.sh @@ -0,0 +1,133 @@ +#!/usr/bin/env bash +# Decision table for the two scratch-dir PreToolUse guards. Run: bash .claude/hooks/test-hooks.sh +# +# What this pins is the `ask` column: a matcher change that turns one into a no-decision drops +# that command's only prompt (see lib-guarded-verb.sh). The wrapper, nested-command and quoted +# rows are the ones that catch it. +set -uo pipefail +H="$(cd "${BASH_SOURCE[0]%/*}" && pwd)" +CWD="$(git -C "$H" rev-parse --show-toplevel)" +OUT="$HOME/not-a-git-tree" # never written to; only the guards' path checks look at it +fails=0 + +run() { # run + local hook="$1" want="$2" cmd="$3" out got + out=$(jq -nc --arg c "$cmd" --arg w "$CWD" \ + '{tool_name:"Bash",tool_input:{command:$c},cwd:$w}' | "$H/$hook" 2>&1) + if [ -z "$out" ]; then + got=none + else + got=$(printf '%s' "$out" | jq -r '.hookSpecificOutput.permissionDecision // "PARSE-ERROR"' 2>/dev/null || echo PARSE-ERROR) + fi + local shown="${cmd//$'\n'/ ⏎ }" + if [ "$got" = "$want" ]; then + printf ' ok %-5s %s\n' "$got" "$shown" + else + printf 'FAIL want=%-5s got=%-5s %s\n %s\n' "$want" "$got" "$shown" "$out" + fails=$((fails + 1)) + fi +} + +echo "== guard-rm-outside-tmp.sh ==" +G=guard-rm-outside-tmp.sh +run $G allow "rm -rf /tmp/scratch/x" +run $G allow "rm -rf /tmp/scratch/*" +run $G allow "rm -rf $CWD/frontend/scratch" +run $G ask "rm -rf /tmp" +run $G ask "rm -rf $OUT" +run $G ask "rm -rf $CWD/.git" +run $G ask "rm -rf $CWD/.claude/hooks" # the guards may not delete themselves +run $G ask "rm $CWD/.claude/settings.json" +run $G ask "rm $CWD/.claude/settings.local.json" +run $G ask "rm -rf $CWD/backend/.env" +run $G ask "rm -rf $CWD/.env.local" +run $G ask "rm -rf $CWD" +run $G ask "rm -rf $CWD/*" +run $G ask "rm -rf /etc/passwd" +run $G ask 'rm -rf "$HOME/x"' +run $G ask "rm -rf /tmp/../$OUT" +run $G ask "ls /tmp && rm -rf /tmp/x" +run $G ask 'echo $(rm -rf /etc)' +run $G ask 'echo `rm -rf /etc`' +run $G ask "{ rm -rf /etc; }" +run $G ask "find . -name x | xargs rm" +run $G ask "timeout 5 rm -rf /tmp/x" +run $G ask "stdbuf -o L rm -rf /etc" +run $G ask "FOO=bar rm -rf /tmp/x" +run $G ask "/bin/rm -rf /tmp/x" +run $G ask "'rm' -rf /etc" +run $G ask 'r\m -rf /etc' +run $G ask "! rm -rf /etc" +run $G ask "if true; then rm -rf /etc; fi" +run $G ask ">/dev/null rm -rf $OUT" +# Data that merely mentions a verb is not a command. Both of these prompted in the field. +run $G none "$(printf 'gh pr create --body "$(cat <<%sEOF%s\ndrop `rm` and `mv` from the ask list\nrm is now guarded here\nEOF\n)"' "'" "'")" +run $G none "$(printf 'claude -p "run these in order:\n1: rm -rf /tmp/a\n2: mv /tmp/b /tmp/c"')" +# A wrapper's own flags and assignments are unbounded, so they may not be charged against the +# scan that looks past it — these run rm and must prompt. +run $G ask "env -i HOME=/tmp PATH=/usr/bin LANG=C USER=root SHELL=/bin/sh rm -rf /etc" +run $G ask "sudo -E -H -u root FOO=1 BAR=2 rm -rf $OUT" +run $G ask "xargs -a f -d d -E e -I {} -L 1 -n 1 rm /etc" +run $G ask "env -u A -u B -u C -u D -u E -u F -u G rm -rf /etc" +run $G ask "sudo -u 'root' rm -rf /etc" +run $G ask "$(printf 'echo hi # cat < f"\nrm -rf /etc\nEOF')" +run $G ask "$(printf 'echo "cat < /tmp/a"\nrm -rf /etc\ntrue')" +run $G ask "$(printf "echo 'cat < /tmp/a\nrm -rf /etc\nEOF' "'" "'")" +run $G none "$(printf 'cat <<%sEOF%s 2>&1 | tee /tmp/a\nrm -rf /etc\nEOF' "'" "'")" +# An unquoted body is expanded before its consumer sees it, so it is code. +run $G ask "$(printf 'cat < /tmp/a\n$(rm -rf /etc)\nEOF')" +run $G ask "$(printf 'cat < /tmp/a\nrm -rf /etc\nEOF')" +# ... but a real command after a heredoc still is one. +run $G ask "$(printf 'cat < /tmp/s.sh\nhello\nEOF\nrm -rf %s' "$OUT")" +run $G ask "$(printf 'echo "a << b"\nrm -rf %s' "$OUT")" +run $G none "git rm frontend/foo.ts" +run $G none 'echo $(ls /tmp)' +run $G none 'grep -rn "rm" backend/' +run $G none "cargo build --release" + +echo +echo "== allow-fileops-in-tmp.sh ==" +A=allow-fileops-in-tmp.sh +run $A allow "mv /tmp/a /tmp/b" +run $A allow "chmod 755 /tmp/a" +run $A allow "cp -r /tmp/a /tmp/b" +run $A allow "tar -xzf /tmp/a.tar.gz -C /tmp/out" +run $A ask "mv /tmp/a $OUT" +run $A ask "mv $CWD/AGENTS.md /tmp/a" +run $A ask "chmod -R 777 $CWD" +run $A ask "ls && mv /tmp/a /tmp/b" +run $A ask 'echo $(mv /tmp/a /etc)' +run $A ask "timeout --signal KILL 5 mv /tmp/a /etc" +run $A ask "time -f FORMAT chmod 777 $OUT" +run $A ask "'mv' /tmp/a /etc" +run $A ask 'ch\mod 777 /etc' +run $A none "$(printf 'claude -p "run these in order:\n1: rm -rf /tmp/a\n2: mv /tmp/b /tmp/c"')" +run $A ask "env -i A=1 B=2 C=3 D=4 E=5 F=6 mv /tmp/a /etc" +run $A none "cp $CWD/AGENTS.md /tmp/a" +run $A none "tar -xzf /tmp/a.tar.gz -C $OUT" +run $A none "cargo build" + +echo +[ "$fails" = 0 ] && echo "ALL PASS" || { echo "$fails FAILURES"; exit 1; } diff --git a/.claude/settings.json b/.claude/settings.json index 641ea70b53..a464ca3719 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -73,10 +73,7 @@ "Edit(**/.env.*)" ], "ask": [ - "Bash(rm:*)", "Bash(rmdir:*)", - "Bash(mv:*)", - "Bash(chmod:*)", "Bash(chown:*)", "Bash(truncate:*)", "Bash(shred:*)", diff --git a/.github/workflows/ai-evals-test.yml b/.github/workflows/ai-evals-test.yml index 3ee7aed876..71e79395f3 100644 --- a/.github/workflows/ai-evals-test.yml +++ b/.github/workflows/ai-evals-test.yml @@ -22,6 +22,7 @@ on: - "frontend/src/lib/userDraft.svelte.ts" - "frontend/src/lib/userDraftDbSyncer.svelte.ts" - "frontend/src/lib/infer.ts" + - "frontend/src/lib/components/sessions/**" - ".github/workflows/ai-evals-test.yml" pull_request: types: [opened, reopened, ready_for_review] @@ -35,6 +36,7 @@ on: - "frontend/src/lib/userDraft.svelte.ts" - "frontend/src/lib/userDraftDbSyncer.svelte.ts" - "frontend/src/lib/infer.ts" + - "frontend/src/lib/components/sessions/**" - ".github/workflows/ai-evals-test.yml" concurrency: @@ -124,6 +126,11 @@ jobs: bun install bun test adapters/ + # Harness code that reaches into the frontend module graph; bun cannot load it. + - name: Run harness unit tests (frontend graph) + working-directory: ./ai_evals + run: bun run test:frontend-graph + - name: Run global AI evals timeout-minutes: 20 working-directory: ./ai_evals diff --git a/.github/workflows/pr-ready-review.yml b/.github/workflows/pr-ready-review.yml index d2acde0dbe..4bfb5b4147 100644 --- a/.github/workflows/pr-ready-review.yml +++ b/.github/workflows/pr-ready-review.yml @@ -167,8 +167,14 @@ jobs: env: EXTRA_PROMPT: ${{ inputs.extra_prompt }} run: | + # prior-comments.md is PR comment text verbatim, and commenting needs no write access. + # With a fixed delimiter, a comment containing a bare `EOF` line closes the block early: + # the step dies, and whatever follows in that comment is read as further environment + # assignments for the rest of this job, which holds the review tokens. Hence a random + # delimiter, per GitHub's guidance for untrusted multiline values. + delimiter="REVIEW_PROMPT_EOF_$(openssl rand -hex 16)" { - echo 'REVIEW_PROMPT<> "$GITHUB_ENV" - name: Automatic PR Review diff --git a/.github/workflows/pr-review-commands.yml b/.github/workflows/pr-review-commands.yml index 3c04e4e5f2..d29ffe6561 100644 --- a/.github/workflows/pr-review-commands.yml +++ b/.github/workflows/pr-review-commands.yml @@ -25,16 +25,20 @@ jobs: REMAINDER_FIRST_LINE=${FIRST_LINE#"$FIRST_WORD"} REMAINDER_FIRST_LINE=${REMAINDER_FIRST_LINE# } REST=$(printf '%s' "$BODY" | tail -n +2) + # The value is the comment body, which anyone can write. A fixed delimiter lets a + # comment close the block early and have the rest of itself read as further step + # outputs, so the delimiter has to be unguessable. + delimiter="EXTRA_EOF_$(openssl rand -hex 16)" { echo "command=$COMMAND" - echo 'extra_prompt<> "$GITHUB_OUTPUT" ;; *) diff --git a/README.md b/README.md index 2d1d4d62ac..d075becce1 100644 --- a/README.md +++ b/README.md @@ -260,7 +260,7 @@ On self-hosted instances, you might want to import all the approved resource typ | NATIVE_MODE | false | Enable native mode: sets NUM_WORKERS=8, rejects non-native jobs (nativets, postgresql, mysql, etc.) | Worker | | SLEEP_QUEUE | 50 | The number of ms to sleep in between the last check for new jobs in the DB. It is multiplied by NUM_WORKERS such that in average, for one worker instance, there is one pull every SLEEP_QUEUE ms. | Worker | | KEEP_JOB_DIR | false | Keep the job directory after the job is done. Useful for debugging. | Worker | -| EXIT_AFTER_N_JOBS | None | Exit the worker process after it has executed that many jobs, so that a supervisor restarts it and no process runs more than that many, bar the steps of a same-worker flow it has started, which it always finishes (set it to 1 for a process per job; jobs handed to a dedicated worker, and the worker's own init and periodic scripts, do not count). For deployments that isolate executions by process lifetime rather than with nsjail; note that a container restart resets the process, not the container filesystem, so caches and `/tmp` survive it. The worker name is then derived from the hostname instead of being random, so the restarted worker keeps its row in the workers list (an agent worker keeps the row but restarts its job count). Use one worker per process: workers of one process share its environment, so the first to reach the limit shuts the others down too. | Worker | +| EXIT_AFTER_N_JOBS | None | Exit the worker process after it has executed that many jobs, so that a supervisor restarts it and no process runs more than that many, bar the steps of a same-worker flow it has started, which it always finishes (set it to 1 for a process per job; jobs handed to a dedicated worker, and the worker's own init and periodic scripts, do not count). Not counting the init and periodic scripts means they run again on every restart: an init script's runtime is added to the latency of every batch of that many jobs, and a periodic script fires once per process start whatever its interval says. The worker's shell in the workers page also starts backed off rather than after the two minutes it otherwise takes, since a process due to be recycled cannot count on living that long: the first command of a session can wait up to 15s, later ones are immediate. For deployments that isolate executions by process lifetime rather than with nsjail; note that a container restart resets the process, not the container filesystem, so caches and `/tmp` survive it. The worker name is then derived from the hostname instead of being random, so the restarted worker keeps its row in the workers list (an agent worker keeps the row but restarts its job count). Use one worker per process: workers of one process share its environment, so the first to reach the limit shuts the others down too. | Worker | | WORKER_SUFFIX | None | Pins the last part of the worker name, which is otherwise random, so that a restarted worker keeps its row in the workers list. Only needed when several worker processes of the same worker group run on one host, since the name is derived from the hostname: give each of them a distinct value, as two processes sharing one must never happen. At most 64 letters, digits and underscores; anything else is refused at startup. | Worker | | LICENSE_KEY (EE only) | None | License key checked at startup for the Enterprise Edition of Windmill | Worker | | SLACK_SIGNING_SECRET | None | The signing secret of your Slack app. See [Slack documentation](https://api.slack.com/authentication/verifying-requests-from-slack) | Server | diff --git a/ai_evals/README.md b/ai_evals/README.md index 08725e8aa8..ee8f730fc3 100644 --- a/ai_evals/README.md +++ b/ai_evals/README.md @@ -150,6 +150,21 @@ Global initial fixtures can also seed `liveEditorDrafts` with `type`, currently open script, flow, or raw app editor so cases can test prompts that refer to "this" or the "current" item. +Global initial fixtures can seed the session's `artifacts` — `{ name, versions: [{ content, +note? }], role?, approvedVersion? }`, oldest version first, so the artifact starts with the +history `list_artifact_versions` reports — and the `previewTabs` open in its side panel, for +cases that run with `runtime.sessionChat: true`. A tab entry names one destination and may +be the `active` one: + +```json +"previewTabs": [{ "artifact": { "name": "Onboarding plan", "version": 2 }, "active": true }] +``` + +`page` (`{ href, label }`) and `item` (`{ kind, path }`) tabs work the same way. Tabs are +driven by the production tab model, so `open_preview`, `get_preview_status` and +`close_page` really open, report and close them, and a `version` is the pin a reader +chose in the artifact's version picker — which only `get_preview_status` reports. + Global initial fixtures can seed `workspace.variables` with `{ path, value, is_secret, description?, labels?, ws_specific? }` entries so cases can read and edit variables that already exist in the workspace. The mock mirrors the real @@ -265,6 +280,10 @@ Typical artifacts by mode: - `history/`: optional tracked pass-rate history written by `run --record`, one JSONL file per mode - `results/`: local benchmark output and artifacts +Harness unit tests run in two lanes: `bun test adapters/` for plain TypeScript, and +`bun run test:frontend-graph` for `*.vitest.ts` files, which exercise adapters built on +frontend code (Svelte runes, SvelteKit aliases) that bun cannot load. + ## Notes - Frontend modes reuse the production frontend chat code through the Vitest bridge. diff --git a/ai_evals/adapters/frontend/core/global/evalArtifactStore.ts b/ai_evals/adapters/frontend/core/global/evalArtifactStore.ts index ed67cce468..207238cc02 100644 --- a/ai_evals/adapters/frontend/core/global/evalArtifactStore.ts +++ b/ai_evals/adapters/frontend/core/global/evalArtifactStore.ts @@ -1,11 +1,67 @@ // SessionArtifactsStore can't run here (bun has no IndexedDB, nor the compiled $state runes), // so mirror only the shape the artifact tools call, not its scoping or race handling. -export const EVAL_SESSION_ID = "eval-session"; -export function createEvalArtifactHelpers() { +// Cases run concurrently in one process and the preview handlers are registered +// process-wide, keyed by session id — so each run needs its own. +let sessionSeq = 0; + +/** An artifact the session already holds when the case starts: history has to predate the + * run, since one prompt cannot both build a past and reason about it. */ +export interface SeededArtifact { + name: string; + role?: "plan"; + /** Which version the user agreed to. Below the last one means the current text is a + * proposal they turned down, which is the state worth seeding. */ + approvedVersion?: number; + /** Oldest first; the last one is the artifact's current content. */ + versions: Array<{ content: string; note?: string }>; +} + +export function createEvalArtifactHelpers(seed: SeededArtifact[] = []) { + const sessionId = `eval-session-${sessionSeq++}`; const items = new Map>(); // Snapshots per artifact id, oldest first — the version tools read history from here. const history = new Map>>(); + // How a preview-tab fixture names the artifact its tab shows. + const seededIds = new Map(); let seq = 0; + for (const entry of seed) { + const id = `eval-artifact-${seq++}`; + const current = entry.versions.at(-1); + if (!current) continue; + // A preview tab names the artifact it shows, so a shared name would open whichever + // one happened to be seeded last. + if (seededIds.has(entry.name)) { + throw new Error( + `Two seeded artifacts are named "${entry.name}" — a preview tab fixture could not tell them apart`, + ); + } + seededIds.set(entry.name, id); + items.set(id, { + id, + sessionId, + chatId: "eval-chat", + kind: "md", + name: entry.name, + content: current.content, + role: entry.role, + approvedVersion: entry.approvedVersion, + createdAt: 0, + updatedAt: seq, + version: entry.versions.length, + }); + history.set( + id, + entry.versions.map((v, i) => ({ + key: `${id}:${i + 1}`, + artifactId: id, + version: i + 1, + name: entry.name, + content: v.content, + savedAt: i, + note: v.note, + })), + ); + } const snapshotOf = ( artifact: Record, version: number, @@ -21,6 +77,16 @@ export function createEvalArtifactHelpers() { }); const store = { create: async (sessionId: string, input: Record) => { + // One plan per session, as SessionArtifactsStore enforces it — the tool refuses + // first, so reaching this means a case drove create_artifact past that message. + if ( + input.role === "plan" && + [...items.values()].some( + (a) => a.sessionId === sessionId && a.role === "plan", + ) + ) { + throw new Error(`Session ${sessionId} already has a plan document`); + } const now = seq++; const artifact = { id: `eval-artifact-${now}`, @@ -29,6 +95,10 @@ export function createEvalArtifactHelpers() { kind: input.kind ?? "md", name: input.name, content: input.content, + // The plan document is only distinguishable by these, both in the snapshot the + // judge reads and in what list_artifacts reports back to the model. + role: input.role, + approvedVersion: input.approvedVersion, createdAt: now, updatedAt: now, version: 1, @@ -58,6 +128,15 @@ export function createEvalArtifactHelpers() { ...existing, name: input.name ?? existing.name, content: input.content ?? existing.content, + // Carried only onto a version this write produced, as SessionArtifactsStore does: + // a rename cannot promote a proposal the user turned down. + approvedVersion: + input.approvedVersion ?? + (input.keepApproved && + existing.approvedVersion !== undefined && + contentChanged + ? version + : existing.approvedVersion), updatedAt: seq++, version, }; @@ -84,10 +163,12 @@ export function createEvalArtifactHelpers() { return { helpers: { artifacts: store, - sessionId: EVAL_SESSION_ID, + sessionId, getChatId: () => "eval-chat", - openArtifact: () => {}, + openArtifact: (_id: string, _name: string) => {}, }, + sessionId, + seededIds, snapshot: () => [...items.values()], }; } diff --git a/ai_evals/adapters/frontend/core/global/evalPreviewTabs.ts b/ai_evals/adapters/frontend/core/global/evalPreviewTabs.ts new file mode 100644 index 0000000000..f32bf1e22e --- /dev/null +++ b/ai_evals/adapters/frontend/core/global/evalPreviewTabs.ts @@ -0,0 +1,188 @@ +import { + setClosePreviewTabsHandler, + setGetPreviewStatusHandler, + setOpenPagePreviewHandler, + setOpenPreviewHandler, +} from "../../../../../frontend/src/lib/components/copilot/chat/global/core"; +import type { GlobalActivePreviewContext } from "../../../../../frontend/src/lib/components/copilot/chat/global/core"; +import { + describePreview, + previewTargetForSessionTarget, + selectPreviewTabsToClose, + SessionPreviewTabs, + whereIs, +} from "../../../../../frontend/src/lib/components/sessions/sessionPreviewTabs.svelte"; +import { + previewLocationContext, + previewLocationLabel, + promptSafe, + resolvePreviewTab, +} from "../../../../../frontend/src/lib/components/sessions/previewRouter"; +import type { ArtifactVersionTarget } from "../../../../../frontend/src/lib/components/sessions/previewRouter"; +import type { SessionTarget } from "../../../../../frontend/src/lib/components/sessions/sessionState.svelte"; + +// The side panel a session chat talks to, driven by the production tab model rather than +// by canned tool results — so a case measures what the real open_preview / get_preview_status +// / close_page report about the tabs the reader has. sessionRuntime.svelte.ts (the production +// owner of these handlers) can't run here: it reaches for IndexedDB, stores and live editors. + +export interface EvalPreviewTabFixture { + /** Artifact tab, named by the artifact fixture it shows. `version` pins it, as a reader does. */ + artifact?: { name: string; version?: number }; + /** Workspace page tab, e.g. `{ href: "/runs", label: "Runs" }`. */ + page?: { href: string; label: string }; + /** Editor tab for a workspace item. */ + item?: { kind: SessionTarget["kind"]; path: string }; + /** Tab the reader is looking at. Defaults to the last seeded one. */ + active?: boolean; +} + +// Registered once for the whole process, as production does at module load, and dispatched +// by session id: global cases run concurrently, so a per-run registration would have every +// case answering out of whichever run registered last. +const panels = new Map(); + +const NO_SESSION = "No active session; the preview panel is unavailable."; + +function panelFor(sessionId: string | undefined): SessionPreviewTabs | undefined { + return sessionId ? panels.get(sessionId) : undefined; +} + +setGetPreviewStatusHandler((sessionId) => { + const owner = panelFor(sessionId); + if (!owner) return NO_SESSION; + return describePreview(owner.tabs, owner.activeId, !!owner.displayedTab); +}); + +setOpenPreviewHandler(async ({ sessionId, kind, path }) => { + const owner = panelFor(sessionId); + if (!owner) return "Error: no active session to open the preview in."; + const target = previewTargetForSessionTarget(kind, path); + if (!target) { + return `Error: ${kind} targets cannot be shown in the preview panel.`; + } + // The pipeline branch of the production handler waits on an editor that only exists once + // a canvas mounts, which never happens here — a pipeline preview reports as any other. + const result = owner.open(target); + return result.status === "focused" + ? `A preview tab is already showing ${kind} "${path}" — focused it.` + : `Opened ${kind} preview for ${path} in a new tab in the side panel.`; +}); + +setOpenPagePreviewHandler(({ sessionId, href, label, newTab }) => { + const owner = panelFor(sessionId); + if (!owner) return undefined; + const result = owner.open({ type: "page", href, label }, { forceNewTab: newTab }); + if (result.status === "focused") { + return `A preview tab is already showing ${label} — focused it.`; + } + if (result.status === "retargeted") { + return `Updated the ${label} preview tab with the requested view.`; + } + return `Opened ${label} in a new preview tab in the side panel.`; +}); + +setClosePreviewTabsHandler(({ sessionId, all, match }) => { + const owner = panelFor(sessionId); + if (!owner) return NO_SESSION; + if (owner.tabs.length === 0) return "The preview panel has no open tabs."; + const labelFor = (t: (typeof owner.tabs)[number]) => + promptSafe(previewLocationLabel(whereIs(t))); + const doomed = selectPreviewTabsToClose(owner.tabs, { all, match }); + if (doomed.length === 0) { + return `No open tab matched "${match}". Open tabs: ${owner.tabs.map(labelFor).join(", ")}.`; + } + const closedLabels = doomed.map(labelFor); + for (const t of doomed) owner.close(t.id); + return `Closed ${closedLabels.length} preview tab${closedLabels.length === 1 ? "" : "s"} (${closedLabels.join(", ")}).`; +}); + +export interface EvalPreviewPanel { + /** Mirrors production: a written artifact is shown in the panel. `version` carries the + * caller's intent for the version picker — `latest` drops a pin the reader had set. */ + openArtifact: (id: string, name: string, version?: ArtifactVersionTarget) => void; + /** What the user message stamps as ACTIVE PREVIEW, as sessionRuntime's resolver reads it. */ + activePreview: () => GlobalActivePreviewContext | undefined; + dispose: () => void; +} + +export function createEvalPreviewPanel(input: { + sessionId: string; + tabs: EvalPreviewTabFixture[]; + /** Artifact ids by name, from the artifact fixture seeding. */ + artifactIds: Map; +}): EvalPreviewPanel { + // Nothing durable to write back to, and no debounce worth waiting on. + const owner = new SessionPreviewTabs( + { tabs: [], activeId: "", collapsed: false }, + { persist: () => {} }, + 0, + ); + // Opening a tab makes it the active one, so the fixture's pick can only be applied once + // every tab is seeded — selecting inside the loop would lose to the next open. + let requestedActive: string | undefined; + for (const fixture of input.tabs) { + const opened = seedTab(owner, fixture, input.artifactIds); + if (opened && fixture.active) requestedActive = opened; + } + if (requestedActive) owner.select(requestedActive); + // Registered last: seeding throws on a malformed fixture, and this map outlives the run. + panels.set(input.sessionId, owner); + + return { + openArtifact: (id, name, version) => { + owner.open({ type: "artifact", id, name, version }); + }, + activePreview: () => { + const tab = owner.displayedTab; + if (!tab) return undefined; + // Artifact and editor tabs are not iframes: they carry no page location, and an + // artifact's pinned version reaches the chat only through get_preview_status. + if (resolvePreviewTab(tab.url).kind !== "iframe") return undefined; + return previewLocationContext(whereIs(tab)); + }, + dispose: () => { + panels.delete(input.sessionId); + }, + }; +} + +// Seeds one tab through the production open path and returns its id, so a fixture cannot +// describe a tab the panel could not have reached on its own. +function seedTab( + owner: SessionPreviewTabs, + fixture: EvalPreviewTabFixture, + artifactIds: Map, +): string | undefined { + // A tab shows one destination; the branches below would silently keep the first. + const named = [fixture.artifact, fixture.page, fixture.item].filter(Boolean); + if (named.length > 1) { + throw new Error( + "Preview tab fixture sets more than one of artifact, page and item — a tab shows one of them", + ); + } + if (fixture.artifact) { + const id = artifactIds.get(fixture.artifact.name); + if (!id) { + throw new Error( + `Preview tab fixture references artifact "${fixture.artifact.name}", which no artifact fixture seeds`, + ); + } + owner.open({ type: "artifact", id, name: fixture.artifact.name }); + // A pin is the reader's own pick in the version picker, never a side effect of opening. + if (fixture.artifact.version !== undefined) { + owner.pinArtifactVersion(id, fixture.artifact.version); + } + } else if (fixture.page) { + owner.open({ type: "page", href: fixture.page.href, label: fixture.page.label }); + } else if (fixture.item) { + const target = previewTargetForSessionTarget(fixture.item.kind, fixture.item.path); + if (!target) { + throw new Error(`Preview tab fixture has an unpreviewable item kind: ${fixture.item.kind}`); + } + owner.open(target); + } else { + throw new Error("Preview tab fixture must set one of artifact, page or item"); + } + return owner.activeId; +} diff --git a/ai_evals/adapters/frontend/core/global/evalPreviewTabs.vitest.ts b/ai_evals/adapters/frontend/core/global/evalPreviewTabs.vitest.ts new file mode 100644 index 0000000000..b9a2990a0a --- /dev/null +++ b/ai_evals/adapters/frontend/core/global/evalPreviewTabs.vitest.ts @@ -0,0 +1,37 @@ +import { expect, it, vi } from 'vitest' + +// The panel pulls in the global tool module, which reaches the editor stack it never uses here. +vi.mock('monaco-editor', () => ({ + editor: {}, + languages: {}, + KeyCode: {}, + Uri: { parse: (value: string) => ({ toString: () => value }) }, + MarkerSeverity: { Error: 8, Warning: 4, Info: 2, Hint: 1 } +})) +vi.mock('@codingame/monaco-vscode-standalone-typescript-language-features', () => ({ + getTypeScriptWorker: async () => async () => ({}), + typescriptVersion: 'test' +})) +vi.mock('@codingame/monaco-vscode-languages-service-override', () => ({ default: () => ({}) })) +vi.mock('$lib/components/vscode', () => ({})) + +const { createEvalPreviewPanel } = await import('./evalPreviewTabs') + +// Every open makes its own tab active, so a fixture's `active` flag only means anything if +// it survives the tabs seeded after it. Lose that and a case still runs — against a panel +// state its author never described. +it('keeps the tab a fixture marks active, not the last one seeded', () => { + const panel = createEvalPreviewPanel({ + sessionId: 'eval-preview-tabs-unit-test', + tabs: [ + { page: { href: '/runs', label: 'Runs' }, active: true }, + { artifact: { name: 'Onboarding plan' } } + ], + artifactIds: new Map([['Onboarding plan', 'eval-artifact-0']]) + }) + try { + expect(panel.activePreview()?.location).toBe('/runs') + } finally { + panel.dispose() + } +}) diff --git a/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts b/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts index dcd771663a..d35d874f71 100644 --- a/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts +++ b/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts @@ -12,9 +12,19 @@ import { getGlobalDraft, listGlobalDrafts, } from "../../../../../frontend/src/lib/components/copilot/chat/global/userDraftAdapter"; +import { appendPlanModeInstructions } from "../../../../../frontend/src/lib/components/copilot/chat/planMode"; import type { Tool as ProductionTool } from "../../../../../frontend/src/lib/components/copilot/chat/shared"; +import { createEvalPlanTools } from "./planModeTools"; import { UserDraft } from "../../../../../frontend/src/lib/userDraft.svelte"; -import { createEvalArtifactHelpers } from "./evalArtifactStore"; +import { + createEvalArtifactHelpers, + type SeededArtifact, +} from "./evalArtifactStore"; +import { + createEvalPreviewPanel, + type EvalPreviewPanel, + type EvalPreviewTabFixture, +} from "./evalPreviewTabs"; import type { ModeRunContext } from "../../../../core/types"; import type { GlobalDraftState } from "../../../../core/validators"; import type { WindmillBackendSettings } from "../../../../core/windmillBackendSettings"; @@ -83,11 +93,18 @@ export interface GlobalEvalOptions { user?: GlobalUserFixture; // Emulate a session chat (preview tools + session prompt); default false = standalone baseline. sessionChat?: boolean; + // Start in plan mode: the gate refuses every tool without `planModeSafe`, and the two plan + // tools are offered. Needs sessionChat, which is what plan mode is gated on in production. + planMode?: boolean; model?: string; maxIterations?: number; provider?: AIProvider; backend: WindmillBackendSettings; workspaceRoot?: string; + // Artifacts the session already holds when the run starts. + artifacts?: SeededArtifact[]; + /** Tabs already open in the side panel, including any artifact version the reader pinned. */ + previewTabs?: EvalPreviewTabFixture[]; runContext?: ModeRunContext; } @@ -106,27 +123,67 @@ export async function runGlobalEval( options.workspaceFixtures ?? {}, ); seedLiveEditorDrafts(workspaceRoot, options.liveEditorDrafts ?? []); + // Declared out here only so `finally` can reach it; a malformed fixture throws while + // building it, and everything seeded above still has to be torn down. + let panel: EvalPreviewPanel | undefined; try { + const evalArtifacts = createEvalArtifactHelpers(options.artifacts); + // Only a session chat has a side panel, so only it gets one here. Seeded tabs would + // otherwise vanish without a word, and the case would measure an empty panel. + if (!options.sessionChat && options.previewTabs?.length) { + throw new Error( + "This fixture seeds previewTabs, which only a session chat has — set runtime.sessionChat: true on the case.", + ); + } + if (options.sessionChat) { + panel = createEvalPreviewPanel({ + sessionId: evalArtifacts.sessionId, + tabs: options.previewTabs ?? [], + artifactIds: evalArtifacts.seededIds, + }); + } const model = options.model ?? "claude-haiku-4-5-20251001"; const injectActiveEditorContext = process.env[DISABLE_ACTIVE_EDITOR_CONTEXT_ENV] !== "1"; + const planMode = options.planMode + ? createEvalPlanTools({ + create: evalArtifacts.helpers.artifacts.create, + sessionId: evalArtifacts.helpers.sessionId, + chatId: evalArtifacts.helpers.getChatId(), + }) + : undefined; // Pass the seeded identity straight to the prompt builder rather than mutating // the process-global `userStore`, so concurrent cases never race on it. - const evalArtifacts = createEvalArtifactHelpers(); + const baseSystemMessage = prepareGlobalSystemMessage(undefined, { + user: options.user, + previewTools: options.sessionChat ?? false, + }); const rawResult = await runEval({ userPrompt, - systemMessage: prepareGlobalSystemMessage(undefined, { - user: options.user, - previewTools: options.sessionChat ?? false, + systemMessage: baseSystemMessage, + // Re-derived per request, as production's getter is: the instructions have to leave + // the prompt when the plan is approved, or the model is still told it may not build + // while the gate has already opened. + getSystemMessage: planMode + ? () => + planMode.isPlanModeActive() + ? appendPlanModeInstructions(baseSystemMessage, 0) + : baseSystemMessage + : undefined, + isPlanModeActive: planMode?.isPlanModeActive, + isToolAvailable: planMode?.isToolAvailable, + userMessage: prepareGlobalUserMessage(userPrompt, [], { + ...(injectActiveEditorContext ? { workspace: workspaceRoot } : {}), + activePreview: panel?.activePreview(), }), - userMessage: prepareGlobalUserMessage( - userPrompt, - [], - injectActiveEditorContext ? { workspace: workspaceRoot } : {}, - ), - tools: getGlobalEvalTools(options.sessionChat ?? false), - helpers: evalArtifacts.helpers, + tools: [ + ...getGlobalEvalTools(options.sessionChat ?? false), + ...(planMode?.tools ?? []), + ], + helpers: panel + ? { ...evalArtifacts.helpers, openArtifact: panel.openArtifact } + : evalArtifacts.helpers, apiKey, getOutput: async () => ({ ...(await collectGlobalDraftState(workspaceRoot)), @@ -159,6 +216,7 @@ export async function runGlobalEval( finalContextTokens: rawResult.finalContextTokens, }; } finally { + panel?.dispose(); clearGlobalDrafts(workspaceRoot); clearLiveEditorDrafts(workspaceRoot, options.liveEditorDrafts ?? []); unregisterBenchmarkWorkspaceRunnables(workspaceRoot); diff --git a/ai_evals/adapters/frontend/core/global/planModeTools.ts b/ai_evals/adapters/frontend/core/global/planModeTools.ts new file mode 100644 index 0000000000..bea5538fc8 --- /dev/null +++ b/ai_evals/adapters/frontend/core/global/planModeTools.ts @@ -0,0 +1,69 @@ +import { + EXIT_PLAN_MODE_TOOL, + EXIT_PLAN_MODE_TOOL_DESCRIPTION, + derivePlanTitle, + exitPlanModeArgs, + planSummaryOf, +} from "../../../../../frontend/src/lib/components/copilot/chat/planMode"; +import { PLAN_MODE_MESSAGES } from "../../../../../frontend/src/lib/components/copilot/chat/planModeMessages"; +import { createToolDef } from "../../../../../frontend/src/lib/components/copilot/chat/shared"; +import type { Tool as ProductionTool } from "../../../../../frontend/src/lib/components/copilot/chat/shared"; + +/** + * `exit_plan_mode` built from the production schema, description and messages, so a case + * exercises the real gate and wording with the posture living here rather than on the + * manager. It resolves immediately — the runners define no `requestConfirmation`, so the + * plan is always approved and a refused one cannot be expressed. + */ +export function createEvalPlanTools(artifacts: { + create: ( + sessionId: string, + input: Record, + ) => Promise<{ id: string; name: string }>; + sessionId: string; + chatId: string; +}): { + tools: ProductionTool<{}>[]; + isPlanModeActive: () => boolean; + isToolAvailable: (name: string) => boolean; +} { + let planActive = true; + return { + isPlanModeActive: () => planActive, + // Withdrawn on approval, as production's tool getter does it: leaving it advertised + // invites a second hand-over of a plan already agreed, which would write a duplicate. + // Production would offer enter_plan_mode in its place; these cases stop at the first + // hand-over, so a fresh planning round belongs to a case of its own. + isToolAvailable: (name) => name !== EXIT_PLAN_MODE_TOOL || planActive, + // Production offers one plan tool at a time and these cases start in plan mode, so + // enter_plan_mode would only invite a turn spent entering a posture already held. + tools: [ + { + def: createToolDef( + exitPlanModeArgs, + EXIT_PLAN_MODE_TOOL, + EXIT_PLAN_MODE_TOOL_DESCRIPTION, + ), + // Carries the safety tag for the same reason production does: it is the only way out + // of the posture, so the gate must not refuse it. + planModeSafe: true, + fn: async ({ args }) => { + const summary = planSummaryOf(args); + if (!summary?.trim()) { + return PLAN_MODE_MESSAGES.missingSummary; + } + planActive = false; + await artifacts.create(artifacts.sessionId, { + name: derivePlanTitle(summary), + content: summary, + kind: "md", + role: "plan", + approvedVersion: 1, + chatId: artifacts.chatId, + }); + return PLAN_MODE_MESSAGES.approvedWithDoc; + }, + }, + ] as ProductionTool<{}>[], + }; +} diff --git a/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts b/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts index f787743dfa..4501f24810 100644 --- a/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts +++ b/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts @@ -43,6 +43,15 @@ export interface RunEvalParams { getOutput: () => TOutput | Promise; /** Model and Windmill backend configuration */ options: EvalRunnerOptions; + /** Drives the production plan-mode gate in processToolCall. Absent leaves it inert, + * which is what every mode but an opted-in global case wants. */ + isPlanModeActive?: () => boolean; + /** Which of `tools` the model is offered on this request. Absent offers all of them. */ + isToolAvailable?: (name: string) => boolean; + /** Re-read before every request, as production's systemMessage getter is. Needed when a + * tool changes what the prompt should say — plan mode's instructions have to come back + * out once the plan is approved. Falls back to the fixed `systemMessage`. */ + getSystemMessage?: () => ChatCompletionSystemMessageParam; onAssistantMessageStart?: () => void; onAssistantToken?: (token: string) => void; onAssistantMessageEnd?: () => void; @@ -68,6 +77,9 @@ export async function runEval( onAssistantToken, onAssistantMessageEnd, onToolCall, + isPlanModeActive, + isToolAvailable, + getSystemMessage, } = params; let shouldEmitMessageStart = true; @@ -119,6 +131,7 @@ export async function runEval( } = { setToolStatus: () => {}, removeToolStatus: () => {}, + isPlanModeActive, onNewToken: (token: string) => { if (shouldEmitMessageStart) { onAssistantMessageStart?.(); @@ -140,8 +153,17 @@ export async function runEval( try { const result = await runChatLoop({ messages, - systemMessage, - tools: wrappedTools, + get systemMessage() { + return getSystemMessage?.() ?? systemMessage; + }, + // Re-derived per request, as `systemMessage` is: a tool the posture has withdrawn + // must leave the schema too, or the model keeps being offered a call the run has + // moved past — and the token counts a case reports include a tool it cannot use. + get tools() { + return isToolAvailable + ? wrappedTools.filter((t) => isToolAvailable(t.def.function.name)) + : wrappedTools; + }, helpers, abortController, callbacks, diff --git a/ai_evals/adapters/frontend/vitest.unit.config.ts b/ai_evals/adapters/frontend/vitest.unit.config.ts new file mode 100644 index 0000000000..7c5a794508 --- /dev/null +++ b/ai_evals/adapters/frontend/vitest.unit.config.ts @@ -0,0 +1,31 @@ +import { fileURLToPath } from 'node:url' +import frontendConfig from '../../../frontend/vite.config.js' + +// Harness unit tests that reach into the frontend module graph. They can't run under +// `bun test` (Svelte runes and the SvelteKit aliases both need this build), so they are +// named `*.vitest.ts` — bun's `*.test.ts` sweep skips them and this config claims them. +const FRONTEND_VITE_CONFIG_PATH = fileURLToPath(new URL('../../../frontend/vite.config.js', import.meta.url)) +const FRONTEND_TEST_SETUP_PATH = fileURLToPath( + new URL('../../../frontend/src/lib/test-setup.ts', import.meta.url) +) +const UNIT_TESTS = fileURLToPath(new URL('./**/*.vitest.ts', import.meta.url)) + +const config = { + ...frontendConfig, + test: { + ...frontendConfig.test, + projects: [ + { + extends: FRONTEND_VITE_CONFIG_PATH, + test: { + name: 'server', + environment: 'node', + include: [UNIT_TESTS], + setupFiles: [FRONTEND_TEST_SETUP_PATH] + } + } + ] + } +} + +export default config diff --git a/ai_evals/cases/global.yaml b/ai_evals/cases/global.yaml index ec2260c0e2..5fb8a1f2c5 100644 --- a/ai_evals/cases/global.yaml +++ b/ai_evals/cases/global.yaml @@ -1180,6 +1180,7 @@ - id: global-closepage1-close-runs-tab prompt: |- You just opened the runs page for me in the side panel. Close that tab, I'm done looking at it. + initial: ai_evals/fixtures/frontend/global/initial/preview_runs_tab.json runtime: maxTurns: 6 sessionChat: true @@ -1233,6 +1234,32 @@ - creates one artifact and revises it rather than creating a second artifact - each revision carries a short description of what changed +# A reader who pins an older version in the artifact's picker is looking at something the +# artifact tools never report: an artifact tab carries no ACTIVE PREVIEW section, so the pin +# reaches the chat through get_preview_status alone. Asked what is on screen, the model has +# to read the panel instead of answering from the artifact's own history. + +- id: global-artifact-pinned-version-question + prompt: |- + Which version of the onboarding plan am I looking at right now? + initial: ai_evals/fixtures/frontend/global/initial/artifact_onboarding_plan_pinned_v2.json + runtime: + maxTurns: 6 + sessionChat: true + validate: + draftCountExactly: 0 + toolExpect: + requiredToolsUsed: + - get_preview_status + forbiddenToolsUsed: + - create_artifact + - update_artifact + - deploy_workspace_item + skipJudge: true + judgeChecklist: + - answers that the panel is showing version 2, not the latest version 5 + - does not edit or re-create the artifact + # --- Documentation search (search_docs) --- # Pure product-knowledge questions: the assistant should consult the docs via # search_docs and answer conversationally, not draft or mutate anything. No @@ -1756,8 +1783,32 @@ judgeChecklist: - saves the plan as a markdown artifact via create_artifact rather than only replying inline - the artifact content has a title, a one-line summary, and three or four bullet steps for onboarding + - the artifact is registered as the session's plan (role "plan"), not as an ordinary note - the user asked for the plan they will come back to and revise - does not create a flow or script draft yet +- id: global-planmode1-hands-over-a-plan + prompt: |- + Our support inbox is a mess. I want incoming emails triaged by urgency and routed to the + right team, with anything urgent also posted to Slack. + Work out how you'd build this in Windmill. + initial: ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json + runtime: + maxTurns: 10 + sessionChat: true + planMode: true + # No draft assertion: approving the plan opens the gate mid-run, and building from there is + # what production asks for, so a draft is not a failure. The gate itself is covered by + # shared.test.ts; what only a real model can show is whether it researches and hands over a + # usable plan instead of guessing at one. + toolExpect: + requiredToolsUsed: + - exit_plan_mode + # Not "saves the plan as an artifact": exit_plan_mode writes it, so the harness would + # satisfy that on every run the tool is called at all — it grades itself, not the model. + judgeChecklist: + - the plan covers classifying an incoming email by urgency, routing it to a team, and posting urgent ones to Slack + - the plan is specific about what would be built in Windmill (a flow and its steps, or the scripts involved) + - id: global-npm1-script-search-package prompt: |- Find a good npm package for parsing RSS/Atom feeds and use it to create a draft Bun script diff --git a/ai_evals/core/types.ts b/ai_evals/core/types.ts index f1655bd18f..96e116023a 100644 --- a/ai_evals/core/types.ts +++ b/ai_evals/core/types.ts @@ -33,6 +33,9 @@ export interface EvalCaseRuntimeSpec { appContext?: EvalCaseRuntimeAppContextSpec; // Global mode: run as a session chat (preview tools + session prompt) vs the standalone chat. sessionChat?: boolean; + // Global session chats: start the case in plan mode, so mutating tools are refused until + // the model hands over a plan with exit_plan_mode. + planMode?: boolean; } export interface FlowValidationSpec { diff --git a/ai_evals/fixtures/frontend/global/initial/artifact_onboarding_plan_pinned_v2.json b/ai_evals/fixtures/frontend/global/initial/artifact_onboarding_plan_pinned_v2.json new file mode 100644 index 0000000000..dd4ab37873 --- /dev/null +++ b/ai_evals/fixtures/frontend/global/initial/artifact_onboarding_plan_pinned_v2.json @@ -0,0 +1,38 @@ +{ + "user": { + "username": "admin", + "is_admin": true + }, + "artifacts": [ + { + "name": "Onboarding plan", + "versions": [ + { + "content": "# Onboarding plan\n\nA staged rollout of the customer onboarding flow.\n\n- Collect the signup form\n- Create the customer record\n- Send the welcome email\n" + }, + { + "content": "# Onboarding plan\n\nA staged rollout of the customer onboarding flow.\n\n- Collect the signup form\n- Verify the company domain\n- Create the customer record\n- Send the welcome email\n", + "note": "Added domain verification" + }, + { + "content": "# Onboarding plan\n\nA staged rollout of the customer onboarding flow.\n\n- Collect the signup form\n- Verify the company domain\n- Create the customer record\n- Send the welcome email\n- Schedule the 7-day check-in\n", + "note": "Added the 7-day check-in" + }, + { + "content": "# Onboarding plan\n\nA staged rollout of the customer onboarding flow.\n\n- Collect the signup form\n- Verify the company domain\n- Create the customer record in the CRM\n- Send the welcome email\n- Schedule the 7-day check-in\n", + "note": "Named the CRM as the record store" + }, + { + "content": "# Onboarding plan\n\nA staged rollout of the customer onboarding flow.\n\n- Collect the signup form\n- Verify the company domain\n- Create the customer record in the CRM\n- Send the welcome email\n- Schedule the 7-day check-in\n- Hand over to the account manager\n", + "note": "Added the account-manager handover" + } + ] + } + ], + "previewTabs": [ + { + "artifact": { "name": "Onboarding plan", "version": 2 }, + "active": true + } + ] +} diff --git a/ai_evals/fixtures/frontend/global/initial/preview_runs_tab.json b/ai_evals/fixtures/frontend/global/initial/preview_runs_tab.json new file mode 100644 index 0000000000..565e5ede37 --- /dev/null +++ b/ai_evals/fixtures/frontend/global/initial/preview_runs_tab.json @@ -0,0 +1,12 @@ +{ + "user": { + "username": "admin", + "is_admin": true + }, + "previewTabs": [ + { + "page": { "href": "/runs", "label": "Runs" }, + "active": true + } + ] +} diff --git a/ai_evals/modes/global.ts b/ai_evals/modes/global.ts index 5628c21d5d..ad93b09a55 100644 --- a/ai_evals/modes/global.ts +++ b/ai_evals/modes/global.ts @@ -6,9 +6,15 @@ import { type GlobalLiveEditorDraftFixture, type GlobalUserFixture, } from "../adapters/frontend/core/global/globalEvalRunner"; +import type { SeededArtifact } from "../adapters/frontend/core/global/evalArtifactStore"; +import type { EvalPreviewTabFixture } from "../adapters/frontend/core/global/evalPreviewTabs"; import type { BenchmarkWorkspaceRunnables } from "../adapters/frontend/mockBackend"; import type { FrontendEvalModelConfig } from "../core/models"; -import type { BenchmarkArtifactFile, GlobalValidationSpec, ModeRunner } from "../core/types"; +import type { + BenchmarkArtifactFile, + GlobalValidationSpec, + ModeRunner, +} from "../core/types"; import { validateGlobalState, type GlobalDraftState } from "../core/validators"; import type { WindmillBackendSettings } from "../core/windmillBackendSettings"; import { getFrontendApiKey } from "./frontendCommon"; @@ -17,6 +23,8 @@ export interface GlobalInitialFixture { workspace?: BenchmarkWorkspaceRunnables; liveEditorDrafts?: GlobalLiveEditorDraftFixture[]; user?: GlobalUserFixture; + artifacts?: SeededArtifact[]; + previewTabs?: EvalPreviewTabFixture[]; } export function createGlobalModeRunner( @@ -41,7 +49,10 @@ export function createGlobalModeRunner( workspaceFixtures: initial?.workspace, liveEditorDrafts: initial?.liveEditorDrafts, user: initial?.user, + artifacts: initial?.artifacts, + previewTabs: initial?.previewTabs, sessionChat: context.evalCase?.runtime?.sessionChat, + planMode: context.evalCase?.runtime?.planMode, maxIterations: context.evalCase?.runtime?.maxTurns, provider: modelConfig.provider, model: modelConfig.model, @@ -81,7 +92,9 @@ export function createGlobalModeRunner( }; } -async function loadGlobalInitialFixture(path: string): Promise { +async function loadGlobalInitialFixture( + path: string, +): Promise { if ((await stat(path)).isDirectory()) { const { initialFrontend, initialBackend, initialDatatables } = await loadAppFixtureForEval(path); @@ -104,14 +117,20 @@ async function loadGlobalInitialFixture(path: string): Promise { +async function loadGlobalExpectedFixture( + path: string, +): Promise { return JSON.parse(await readFile(path, "utf8")) as GlobalDraftState; } diff --git a/ai_evals/package.json b/ai_evals/package.json index b7fbeaa1a5..6720c910cb 100644 --- a/ai_evals/package.json +++ b/ai_evals/package.json @@ -4,7 +4,8 @@ "type": "module", "scripts": { "cli": "bun cli/index.ts", - "typecheck": "tsc -p tsconfig.json" + "typecheck": "tsc -p tsconfig.json", + "test:frontend-graph": "cd ../frontend && node_modules/.bin/vitest run --project server --config ../ai_evals/adapters/frontend/vitest.unit.config.ts" }, "dependencies": { "@anthropic-ai/claude-agent-sdk": "^0.2.25", diff --git a/ai_evals/tsconfig.json b/ai_evals/tsconfig.json index 7b06d5788a..50b2b1110d 100644 --- a/ai_evals/tsconfig.json +++ b/ai_evals/tsconfig.json @@ -14,6 +14,8 @@ ], "exclude": [ "./**/*.test.ts", - "./adapters/frontend/vitest.config.ts" + "./**/*.vitest.ts", + "./adapters/frontend/vitest.config.ts", + "./adapters/frontend/vitest.unit.config.ts" ] } diff --git a/backend/.sqlx/query-a41c4cbaffdb714e4a963557de5a4011744d684eb24e03cb4beae6a512613159.json b/backend/.sqlx/query-0c18351237816fe0c56e23801fcb8e70dbffcf08ed121e55c871f727c4ddf626.json similarity index 80% rename from backend/.sqlx/query-a41c4cbaffdb714e4a963557de5a4011744d684eb24e03cb4beae6a512613159.json rename to backend/.sqlx/query-0c18351237816fe0c56e23801fcb8e70dbffcf08ed121e55c871f727c4ddf626.json index 0141c379b4..ceb05001e4 100644 --- a/backend/.sqlx/query-a41c4cbaffdb714e4a963557de5a4011744d684eb24e03cb4beae6a512613159.json +++ b/backend/.sqlx/query-0c18351237816fe0c56e23801fcb8e70dbffcf08ed121e55c871f727c4ddf626.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "UPDATE worker_ping SET ping_at = now(), jobs_executed = $1, custom_tags = $2,\n occupancy_rate = $3, memory_usage = $4, wm_memory_usage = $5, vcpus = COALESCE($7, vcpus),\n memory = COALESCE($8, memory), occupancy_rate_15s = $9, occupancy_rate_5m = $10, occupancy_rate_30m = $11, native_mode = $12 WHERE worker = $6", + "query": "UPDATE worker_ping SET ping_at = now(), jobs_executed = $1, custom_tags = $2,\n occupancy_rate = $3, memory_usage = $4, wm_memory_usage = $5, vcpus = COALESCE($7, vcpus),\n memory = COALESCE($8, memory), occupancy_rate_15s = $9, occupancy_rate_5m = $10, occupancy_rate_30m = $11, native_mode = $12, ip = COALESCE($13, ip) WHERE worker = $6", "describe": { "columns": [], "parameters": { @@ -16,10 +16,11 @@ "Float4", "Float4", "Float4", - "Bool" + "Bool", + "Varchar" ] }, "nullable": [] }, - "hash": "a41c4cbaffdb714e4a963557de5a4011744d684eb24e03cb4beae6a512613159" + "hash": "0c18351237816fe0c56e23801fcb8e70dbffcf08ed121e55c871f727c4ddf626" } diff --git a/backend/.sqlx/query-1d346a14ad5586af347b8e7ac413500a39efa20e4915ffa56fd40537597db36e.json b/backend/.sqlx/query-1d346a14ad5586af347b8e7ac413500a39efa20e4915ffa56fd40537597db36e.json new file mode 100644 index 0000000000..e8ffcfb95b --- /dev/null +++ b/backend/.sqlx/query-1d346a14ad5586af347b8e7ac413500a39efa20e4915ffa56fd40537597db36e.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT 'schedule' AS \"kind!\", COUNT(*)::BIGINT AS \"count!\" FROM schedule\n UNION ALL SELECT 'http', COUNT(*)::BIGINT FROM http_trigger\n UNION ALL SELECT 'websocket', COUNT(*)::BIGINT FROM websocket_trigger\n UNION ALL SELECT 'kafka', COUNT(*)::BIGINT FROM kafka_trigger\n UNION ALL SELECT 'nats', COUNT(*)::BIGINT FROM nats_trigger\n UNION ALL SELECT 'postgres', COUNT(*)::BIGINT FROM postgres_trigger\n UNION ALL SELECT 'mqtt', COUNT(*)::BIGINT FROM mqtt_trigger\n UNION ALL SELECT 'sqs', COUNT(*)::BIGINT FROM sqs_trigger\n UNION ALL SELECT 'gcp', COUNT(*)::BIGINT FROM gcp_trigger\n UNION ALL SELECT 'azure', COUNT(*)::BIGINT FROM azure_trigger\n UNION ALL SELECT 'amqp', COUNT(*)::BIGINT FROM amqp_trigger\n UNION ALL SELECT 'email', COUNT(*)::BIGINT FROM email_trigger\n -- Grouped, not a single 'native' key: these fire as nextcloud/google/github,\n -- so a lone key would not line up with the `trigger`/`fired` series.\n UNION ALL SELECT service_name::text, COUNT(*)::BIGINT FROM native_trigger GROUP BY service_name\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "kind!", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null, + null + ] + }, + "hash": "1d346a14ad5586af347b8e7ac413500a39efa20e4915ffa56fd40537597db36e" +} diff --git a/backend/.sqlx/query-2132a715995f2775917c59c01ba66f2e472a2347f25cc7109f36a805806ee6e2.json b/backend/.sqlx/query-2132a715995f2775917c59c01ba66f2e472a2347f25cc7109f36a805806ee6e2.json deleted file mode 100644 index 673b8f4574..0000000000 --- a/backend/.sqlx/query-2132a715995f2775917c59c01ba66f2e472a2347f25cc7109f36a805806ee6e2.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT username, added_via\n FROM usr\n WHERE workspace_id = $1 AND email = $2\n AND added_via->>'source' = 'instance_group'\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "username", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "added_via", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - false, - true - ] - }, - "hash": "2132a715995f2775917c59c01ba66f2e472a2347f25cc7109f36a805806ee6e2" -} diff --git a/backend/.sqlx/query-9530b234b2ac8f7360552be5e6bb25270e6b4185ce5c7f3c6edd9fbaabf77544.json b/backend/.sqlx/query-236028886d13526daa184f9e6d0a4b2ae43fbaf7f0bbbc8246c5e2d73b6f6aee.json similarity index 50% rename from backend/.sqlx/query-9530b234b2ac8f7360552be5e6bb25270e6b4185ce5c7f3c6edd9fbaabf77544.json rename to backend/.sqlx/query-236028886d13526daa184f9e6d0a4b2ae43fbaf7f0bbbc8246c5e2d73b6f6aee.json index 25a898e2f1..6a43b44e46 100644 --- a/backend/.sqlx/query-9530b234b2ac8f7360552be5e6bb25270e6b4185ce5c7f3c6edd9fbaabf77544.json +++ b/backend/.sqlx/query-236028886d13526daa184f9e6d0a4b2ae43fbaf7f0bbbc8246c5e2d73b6f6aee.json @@ -1,11 +1,11 @@ { "db_name": "PostgreSQL", - "query": "SELECT DISTINCT email FROM usr WHERE added_via->>'source' = 'instance_group' AND added_via->>'group' = $1", + "query": "SELECT instance_role FROM instance_group WHERE name = $1 FOR UPDATE", "describe": { "columns": [ { "ordinal": 0, - "name": "email", + "name": "instance_role", "type_info": "Varchar" } ], @@ -15,8 +15,8 @@ ] }, "nullable": [ - false + true ] }, - "hash": "9530b234b2ac8f7360552be5e6bb25270e6b4185ce5c7f3c6edd9fbaabf77544" + "hash": "236028886d13526daa184f9e6d0a4b2ae43fbaf7f0bbbc8246c5e2d73b6f6aee" } diff --git a/backend/.sqlx/query-255d37bb63595ebfcc61582d0b5e265b861b8b4d650435533e90eeb1d5ee3a68.json b/backend/.sqlx/query-255d37bb63595ebfcc61582d0b5e265b861b8b4d650435533e90eeb1d5ee3a68.json new file mode 100644 index 0000000000..64b6d70542 --- /dev/null +++ b/backend/.sqlx/query-255d37bb63595ebfcc61582d0b5e265b861b8b4d650435533e90eeb1d5ee3a68.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*)::BIGINT AS \"count!\" FROM flow\n WHERE archived = false AND pg_column_size(value) >= $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Int4" + ] + }, + "nullable": [ + null + ] + }, + "hash": "255d37bb63595ebfcc61582d0b5e265b861b8b4d650435533e90eeb1d5ee3a68" +} diff --git a/backend/.sqlx/query-c7c0b7f760f9616ec4a18a8916226b65990f5055c1bc25eedefd303be75f553f.json b/backend/.sqlx/query-374863f06094e404523e700df832243894e40a081fd2e7f95b466612aed054de.json similarity index 65% rename from backend/.sqlx/query-c7c0b7f760f9616ec4a18a8916226b65990f5055c1bc25eedefd303be75f553f.json rename to backend/.sqlx/query-374863f06094e404523e700df832243894e40a081fd2e7f95b466612aed054de.json index 3b175bb9b8..f8b19be3c5 100644 --- a/backend/.sqlx/query-c7c0b7f760f9616ec4a18a8916226b65990f5055c1bc25eedefd303be75f553f.json +++ b/backend/.sqlx/query-374863f06094e404523e700df832243894e40a081fd2e7f95b466612aed054de.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT workspace_id FROM workspace_settings WHERE auto_invite->'instance_groups' IS NOT NULL AND auto_invite->'instance_groups' ? $1", + "query": "SELECT workspace_id FROM workspace_settings WHERE auto_invite->'instance_groups' ?| $1", "describe": { "columns": [ { @@ -11,12 +11,12 @@ ], "parameters": { "Left": [ - "Text" + "TextArray" ] }, "nullable": [ false ] }, - "hash": "c7c0b7f760f9616ec4a18a8916226b65990f5055c1bc25eedefd303be75f553f" + "hash": "374863f06094e404523e700df832243894e40a081fd2e7f95b466612aed054de" } diff --git a/backend/.sqlx/query-3b0eb0571f287eb64c84cf2c8303401af7b78d0fb6cbaa10ccb90769320c8c7a.json b/backend/.sqlx/query-3b0eb0571f287eb64c84cf2c8303401af7b78d0fb6cbaa10ccb90769320c8c7a.json new file mode 100644 index 0000000000..ca79568bb0 --- /dev/null +++ b/backend/.sqlx/query-3b0eb0571f287eb64c84cf2c8303401af7b78d0fb6cbaa10ccb90769320c8c7a.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n COUNT(*) FILTER (WHERE slack_command_script IS NOT NULL)::BIGINT AS \"slack!\",\n COUNT(*) FILTER (WHERE teams_command_script IS NOT NULL)::BIGINT AS \"teams!\"\n FROM workspace_settings", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "slack!", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "teams!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null, + null + ] + }, + "hash": "3b0eb0571f287eb64c84cf2c8303401af7b78d0fb6cbaa10ccb90769320c8c7a" +} diff --git a/backend/.sqlx/query-3bd4f38a1629a69ddda622b6b436198b47c2fe1a507358d49f05031e7beedab6.json b/backend/.sqlx/query-3bd4f38a1629a69ddda622b6b436198b47c2fe1a507358d49f05031e7beedab6.json deleted file mode 100644 index 0c71ee8ad2..0000000000 --- a/backend/.sqlx/query-3bd4f38a1629a69ddda622b6b436198b47c2fe1a507358d49f05031e7beedab6.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT workspace_id,\n auto_invite->'instance_groups_roles' as instance_groups_roles,\n auto_invite->'instance_groups' as instance_groups_json\n FROM workspace_settings\n WHERE auto_invite->'instance_groups' ? $1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "workspace_id", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "instance_groups_roles", - "type_info": "Jsonb" - }, - { - "ordinal": 2, - "name": "instance_groups_json", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - false, - null, - null - ] - }, - "hash": "3bd4f38a1629a69ddda622b6b436198b47c2fe1a507358d49f05031e7beedab6" -} diff --git a/backend/.sqlx/query-5d160ba4958583f1ad42de846c544d8d8e81e1b54925a0c5f2cedc1817d99a1b.json b/backend/.sqlx/query-4348832b4b99021b19a752a0375f7728fb10035ff4a71b8e0242e1184d192a32.json similarity index 58% rename from backend/.sqlx/query-5d160ba4958583f1ad42de846c544d8d8e81e1b54925a0c5f2cedc1817d99a1b.json rename to backend/.sqlx/query-4348832b4b99021b19a752a0375f7728fb10035ff4a71b8e0242e1184d192a32.json index 8fcffe364a..b6342d9b36 100644 --- a/backend/.sqlx/query-5d160ba4958583f1ad42de846c544d8d8e81e1b54925a0c5f2cedc1817d99a1b.json +++ b/backend/.sqlx/query-4348832b4b99021b19a752a0375f7728fb10035ff4a71b8e0242e1184d192a32.json @@ -1,11 +1,11 @@ { "db_name": "PostgreSQL", - "query": "SELECT igroup FROM email_to_igroup WHERE email = $1", + "query": "SELECT name FROM instance_group WHERE name = $1 FOR UPDATE", "describe": { "columns": [ { "ordinal": 0, - "name": "igroup", + "name": "name", "type_info": "Varchar" } ], @@ -18,5 +18,5 @@ false ] }, - "hash": "5d160ba4958583f1ad42de846c544d8d8e81e1b54925a0c5f2cedc1817d99a1b" + "hash": "4348832b4b99021b19a752a0375f7728fb10035ff4a71b8e0242e1184d192a32" } diff --git a/backend/.sqlx/query-43a689277803e5e2204e10263a5749675652c23a231fce65257b053b6faad231.json b/backend/.sqlx/query-43a689277803e5e2204e10263a5749675652c23a231fce65257b053b6faad231.json new file mode 100644 index 0000000000..86af8f2b7b --- /dev/null +++ b/backend/.sqlx/query-43a689277803e5e2204e10263a5749675652c23a231fce65257b053b6faad231.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT name FROM instance_group WHERE name = ANY($1) ORDER BY name FOR UPDATE", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "TextArray" + ] + }, + "nullable": [ + false + ] + }, + "hash": "43a689277803e5e2204e10263a5749675652c23a231fce65257b053b6faad231" +} diff --git a/backend/.sqlx/query-472f224dc9e17d2c50cb9db13c34cc1fb9adb6d0ea36cf223541adb7cac17bdd.json b/backend/.sqlx/query-472f224dc9e17d2c50cb9db13c34cc1fb9adb6d0ea36cf223541adb7cac17bdd.json deleted file mode 100644 index b60ecae184..0000000000 --- a/backend/.sqlx/query-472f224dc9e17d2c50cb9db13c34cc1fb9adb6d0ea36cf223541adb7cac17bdd.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT workspace_id, username, email\n FROM usr\n WHERE email = $1\n AND added_via->>'source' = 'instance_group'\n AND added_via->>'group' = $2\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "workspace_id", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "username", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "email", - "type_info": "Varchar" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - false, - false, - false - ] - }, - "hash": "472f224dc9e17d2c50cb9db13c34cc1fb9adb6d0ea36cf223541adb7cac17bdd" -} diff --git a/backend/.sqlx/query-49bf26ae4b7e3421507f9e7e42c59ad7e0f481a9550e0f70e1145d5e541bd6e5.json b/backend/.sqlx/query-49bf26ae4b7e3421507f9e7e42c59ad7e0f481a9550e0f70e1145d5e541bd6e5.json new file mode 100644 index 0000000000..005ac90b79 --- /dev/null +++ b/backend/.sqlx/query-49bf26ae4b7e3421507f9e7e42c59ad7e0f481a9550e0f70e1145d5e541bd6e5.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT auto_invite->'instance_groups' as \"groups: serde_json::Value\",\n auto_invite->'instance_groups_roles' as \"roles: serde_json::Value\"\n FROM workspace_settings WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "groups: serde_json::Value", + "type_info": "Jsonb" + }, + { + "ordinal": 1, + "name": "roles: serde_json::Value", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null, + null + ] + }, + "hash": "49bf26ae4b7e3421507f9e7e42c59ad7e0f481a9550e0f70e1145d5e541bd6e5" +} diff --git a/backend/.sqlx/query-5344f222417c28efd4f724cbd83382fc69a223dfbb91ab40df895ab60d0f6228.json b/backend/.sqlx/query-5344f222417c28efd4f724cbd83382fc69a223dfbb91ab40df895ab60d0f6228.json new file mode 100644 index 0000000000..08c08bbf75 --- /dev/null +++ b/backend/.sqlx/query-5344f222417c28efd4f724cbd83382fc69a223dfbb91ab40df895ab60d0f6228.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT trigger_kind::text AS \"kind!\", COUNT(*)::BIGINT AS \"count!\"\n FROM capture WHERE created_at > now() - interval '30 days' GROUP BY trigger_kind", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "kind!", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null, + null + ] + }, + "hash": "5344f222417c28efd4f724cbd83382fc69a223dfbb91ab40df895ab60d0f6228" +} diff --git a/backend/.sqlx/query-66e2f8468ba64f22b7a7caa18639d7c833ac2ec573bd89d878b5c8b1afc74d3a.json b/backend/.sqlx/query-66e2f8468ba64f22b7a7caa18639d7c833ac2ec573bd89d878b5c8b1afc74d3a.json deleted file mode 100644 index 15681aab51..0000000000 --- a/backend/.sqlx/query-66e2f8468ba64f22b7a7caa18639d7c833ac2ec573bd89d878b5c8b1afc74d3a.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT\n workspace_id,\n auto_invite->'instance_groups_roles' as instance_groups_roles,\n auto_invite->'instance_groups' as instance_groups_json\n FROM workspace_settings\n WHERE\n auto_invite->'instance_groups' IS NOT NULL\n AND auto_invite->'instance_groups' ? $1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "workspace_id", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "instance_groups_roles", - "type_info": "Jsonb" - }, - { - "ordinal": 2, - "name": "instance_groups_json", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - false, - null, - null - ] - }, - "hash": "66e2f8468ba64f22b7a7caa18639d7c833ac2ec573bd89d878b5c8b1afc74d3a" -} diff --git a/backend/.sqlx/query-6c6b4bd4bd19878fce25d3a8a5ee02686b358b616c318e4bfca084d96b38f1c4.json b/backend/.sqlx/query-6c6b4bd4bd19878fce25d3a8a5ee02686b358b616c318e4bfca084d96b38f1c4.json new file mode 100644 index 0000000000..6576d06ffa --- /dev/null +++ b/backend/.sqlx/query-6c6b4bd4bd19878fce25d3a8a5ee02686b358b616c318e4bfca084d96b38f1c4.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT config AS \"config!\" FROM config WHERE name LIKE 'worker__%' AND config IS NOT NULL", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "config!", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + true + ] + }, + "hash": "6c6b4bd4bd19878fce25d3a8a5ee02686b358b616c318e4bfca084d96b38f1c4" +} diff --git a/backend/.sqlx/query-6fda4517a72b25b0eab47bc69127ee45467d4d45239fb556534429b03442b27b.json b/backend/.sqlx/query-6fda4517a72b25b0eab47bc69127ee45467d4d45239fb556534429b03442b27b.json new file mode 100644 index 0000000000..17641bc8f7 --- /dev/null +++ b/backend/.sqlx/query-6fda4517a72b25b0eab47bc69127ee45467d4d45239fb556534429b03442b27b.json @@ -0,0 +1,46 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT username, email, is_admin, operator,\n added_via->>'group' as granting_group\n FROM usr\n WHERE workspace_id = $1 AND added_via->>'source' = 'instance_group'\n ORDER BY email", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "username", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "is_admin", + "type_info": "Bool" + }, + { + "ordinal": 3, + "name": "operator", + "type_info": "Bool" + }, + { + "ordinal": 4, + "name": "granting_group", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false, + null + ] + }, + "hash": "6fda4517a72b25b0eab47bc69127ee45467d4d45239fb556534429b03442b27b" +} diff --git a/backend/.sqlx/query-75e740531bf794a8568350348253612cfbdecb9fb9cf768f431d4dbd4cd56cfb.json b/backend/.sqlx/query-75e740531bf794a8568350348253612cfbdecb9fb9cf768f431d4dbd4cd56cfb.json deleted file mode 100644 index 3445985466..0000000000 --- a/backend/.sqlx/query-75e740531bf794a8568350348253612cfbdecb9fb9cf768f431d4dbd4cd56cfb.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT auto_invite->'instance_groups' FROM workspace_settings WHERE workspace_id = $1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "75e740531bf794a8568350348253612cfbdecb9fb9cf768f431d4dbd4cd56cfb" -} diff --git a/backend/.sqlx/query-772dc28e57666282d8993268843d3a87e8899b968827251745f998e4dc25863a.json b/backend/.sqlx/query-772dc28e57666282d8993268843d3a87e8899b968827251745f998e4dc25863a.json new file mode 100644 index 0000000000..d5659d725b --- /dev/null +++ b/backend/.sqlx/query-772dc28e57666282d8993268843d3a87e8899b968827251745f998e4dc25863a.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO feature_usage (feature, kind, key, value)\n SELECT * FROM UNNEST($1::text[], $2::text[], $3::text[], $4::bigint[])\n ON CONFLICT (feature, kind, key, entity_id, day)\n DO UPDATE SET value = feature_usage.value + EXCLUDED.value, updated_at = now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "TextArray", + "TextArray", + "TextArray", + "Int8Array" + ] + }, + "nullable": [] + }, + "hash": "772dc28e57666282d8993268843d3a87e8899b968827251745f998e4dc25863a" +} diff --git a/backend/.sqlx/query-2ee6d24b95cdda151585dcff19f8e7c931785fc21f7bbe9c3a82671943ced0ea.json b/backend/.sqlx/query-7947ffe31b8e6f4a38fba9d9caf434fd409ea2806a7c720c2b1d76b2e7db6c32.json similarity index 65% rename from backend/.sqlx/query-2ee6d24b95cdda151585dcff19f8e7c931785fc21f7bbe9c3a82671943ced0ea.json rename to backend/.sqlx/query-7947ffe31b8e6f4a38fba9d9caf434fd409ea2806a7c720c2b1d76b2e7db6c32.json index 6acb31666c..e69a61e65d 100644 --- a/backend/.sqlx/query-2ee6d24b95cdda151585dcff19f8e7c931785fc21f7bbe9c3a82671943ced0ea.json +++ b/backend/.sqlx/query-7947ffe31b8e6f4a38fba9d9caf434fd409ea2806a7c720c2b1d76b2e7db6c32.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "UPDATE schedule SET enabled = false, error = $1 WHERE workspace_id = $2 AND path = $3", + "query": "UPDATE schedule SET enabled = false, error = $1 WHERE workspace_id = $2 AND path = $3 AND enabled = true", "describe": { "columns": [], "parameters": { @@ -12,5 +12,5 @@ }, "nullable": [] }, - "hash": "2ee6d24b95cdda151585dcff19f8e7c931785fc21f7bbe9c3a82671943ced0ea" + "hash": "7947ffe31b8e6f4a38fba9d9caf434fd409ea2806a7c720c2b1d76b2e7db6c32" } diff --git a/backend/.sqlx/query-7b3eadb62ddd07e5e12eb8b5150ddce33008b61bbdab4006194ca7ef5b754802.json b/backend/.sqlx/query-7b3eadb62ddd07e5e12eb8b5150ddce33008b61bbdab4006194ca7ef5b754802.json new file mode 100644 index 0000000000..6ca02417f3 --- /dev/null +++ b/backend/.sqlx/query-7b3eadb62ddd07e5e12eb8b5150ddce33008b61bbdab4006194ca7ef5b754802.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n (SELECT COUNT(*) FROM script\n WHERE dedicated_worker = true AND archived = false AND deleted = false)::BIGINT AS \"scripts!\",\n (SELECT COUNT(*) FROM flow WHERE dedicated_worker = true AND archived = false)::BIGINT AS \"flows!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "scripts!", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "flows!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null, + null + ] + }, + "hash": "7b3eadb62ddd07e5e12eb8b5150ddce33008b61bbdab4006194ca7ef5b754802" +} diff --git a/backend/.sqlx/query-f582cac90b4b7d732956b74eebc51323ef8acd3f627e7517451fcc72998d22bc.json b/backend/.sqlx/query-815c5e8fd91dc119a803f4f1f56b8016bdaabfa2ff61a1abe8f73f5a7336ab61.json similarity index 64% rename from backend/.sqlx/query-f582cac90b4b7d732956b74eebc51323ef8acd3f627e7517451fcc72998d22bc.json rename to backend/.sqlx/query-815c5e8fd91dc119a803f4f1f56b8016bdaabfa2ff61a1abe8f73f5a7336ab61.json index 5bfc4710c8..fc25b16733 100644 --- a/backend/.sqlx/query-f582cac90b4b7d732956b74eebc51323ef8acd3f627e7517451fcc72998d22bc.json +++ b/backend/.sqlx/query-815c5e8fd91dc119a803f4f1f56b8016bdaabfa2ff61a1abe8f73f5a7336ab61.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "UPDATE usr SET added_via = $1 WHERE workspace_id = $2 AND email = $3", + "query": "UPDATE usr SET added_via = $1 WHERE workspace_id = $2 AND email = $3 AND added_via->>'source' = 'instance_group'", "describe": { "columns": [], "parameters": { @@ -12,5 +12,5 @@ }, "nullable": [] }, - "hash": "f582cac90b4b7d732956b74eebc51323ef8acd3f627e7517451fcc72998d22bc" + "hash": "815c5e8fd91dc119a803f4f1f56b8016bdaabfa2ff61a1abe8f73f5a7336ab61" } diff --git a/backend/.sqlx/query-83ec97f6aad154e0e06ee05a3647dab8f89b1b2d7a569c7eac8c4169e37b9f8b.json b/backend/.sqlx/query-83ec97f6aad154e0e06ee05a3647dab8f89b1b2d7a569c7eac8c4169e37b9f8b.json deleted file mode 100644 index 4e0bc23b69..0000000000 --- a/backend/.sqlx/query-83ec97f6aad154e0e06ee05a3647dab8f89b1b2d7a569c7eac8c4169e37b9f8b.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT value FROM global_settings WHERE name = 'smtp_settings'", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "value", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false - ] - }, - "hash": "83ec97f6aad154e0e06ee05a3647dab8f89b1b2d7a569c7eac8c4169e37b9f8b" -} diff --git a/backend/.sqlx/query-84fcddaf5bc61d607a6e6e5e31de7436b203a3baa7ef0509cb8e6c52270ae3a9.json b/backend/.sqlx/query-84fcddaf5bc61d607a6e6e5e31de7436b203a3baa7ef0509cb8e6c52270ae3a9.json new file mode 100644 index 0000000000..557466e8b7 --- /dev/null +++ b/backend/.sqlx/query-84fcddaf5bc61d607a6e6e5e31de7436b203a3baa7ef0509cb8e6c52270ae3a9.json @@ -0,0 +1,35 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH RECURSIVE chain(id, parent_job) AS (\n SELECT id, parent_job FROM v2_job WHERE id = $1 AND workspace_id = $2\n UNION ALL\n SELECT j.id, j.parent_job FROM v2_job j\n JOIN chain c ON j.id = c.parent_job AND j.workspace_id = $2\n )\n SELECT j.runnable_path,\n CASE\n WHEN j.kind IN ('script', 'script_hub', 'unassigned_script') THEN 'scripts'\n WHEN j.kind IN ('flow', 'unassigned_flow') THEN 'flows'\n WHEN j.kind IN ('singlestepflow', 'unassigned_singlestepflow') THEN\n CASE WHEN COALESCE(\n (SELECT m->'value'->>'type'\n FROM jsonb_array_elements(j.raw_flow->'modules') m\n WHERE m->>'id' IN ('a', 'main')\n LIMIT 1),\n 'script'\n ) = 'flow' THEN 'flows' ELSE 'scripts' END\n END AS scope_kind,\n CASE WHEN j.trigger_kind = 'app' THEN j.trigger END AS launched_by_app\n FROM v2_job j JOIN chain c ON c.id = j.id\n WHERE j.workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "runnable_path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "scope_kind", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "launched_by_app", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + true, + null, + null + ] + }, + "hash": "84fcddaf5bc61d607a6e6e5e31de7436b203a3baa7ef0509cb8e6c52270ae3a9" +} diff --git a/backend/.sqlx/query-928767710fb8b7dc0b1edc897d8ce9b6b59ae2f63e684d3db6eecbcffd767711.json b/backend/.sqlx/query-928767710fb8b7dc0b1edc897d8ce9b6b59ae2f63e684d3db6eecbcffd767711.json new file mode 100644 index 0000000000..36e067a79a --- /dev/null +++ b/backend/.sqlx/query-928767710fb8b7dc0b1edc897d8ce9b6b59ae2f63e684d3db6eecbcffd767711.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO trigger_history\n (workspace_id, trigger_kind, path, operation, source, username, changes)\n SELECT $1, $2, p, $3, $4, $5, $6 FROM unnest($7::text[]) p", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Jsonb", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "928767710fb8b7dc0b1edc897d8ce9b6b59ae2f63e684d3db6eecbcffd767711" +} diff --git a/backend/.sqlx/query-9d9fbcb598c582a29d65be87a1e35baa410f93be3cac4b8dfd33ddbef446d3fe.json b/backend/.sqlx/query-9d9fbcb598c582a29d65be87a1e35baa410f93be3cac4b8dfd33ddbef446d3fe.json deleted file mode 100644 index 2c95e06391..0000000000 --- a/backend/.sqlx/query-9d9fbcb598c582a29d65be87a1e35baa410f93be3cac4b8dfd33ddbef446d3fe.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, dedicated_workers, wm_version, vcpus, memory, job_isolation, native_mode) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) ON CONFLICT (worker)\n DO UPDATE set ping_at = now(), worker_instance = EXCLUDED.worker_instance, ip = EXCLUDED.ip, custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group, dedicated_worker = EXCLUDED.dedicated_worker, dedicated_workers = EXCLUDED.dedicated_workers, wm_version = EXCLUDED.wm_version, vcpus = COALESCE(EXCLUDED.vcpus, worker_ping.vcpus), memory = COALESCE(EXCLUDED.memory, worker_ping.memory), job_isolation = EXCLUDED.job_isolation, native_mode = EXCLUDED.native_mode, current_job_id = NULL, current_job_workspace_id = NULL\n RETURNING jobs_executed", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "jobs_executed", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Varchar", - "Varchar", - "Varchar", - "TextArray", - "Varchar", - "Varchar", - "TextArray", - "Varchar", - "Int8", - "Int8", - "Text", - "Bool" - ] - }, - "nullable": [ - false - ] - }, - "hash": "9d9fbcb598c582a29d65be87a1e35baa410f93be3cac4b8dfd33ddbef446d3fe" -} diff --git a/backend/.sqlx/query-a2f047f9ca4b8a47c985fa092ba0d2dc54f7169af1c940b84345c477865ae82c.json b/backend/.sqlx/query-a2f047f9ca4b8a47c985fa092ba0d2dc54f7169af1c940b84345c477865ae82c.json new file mode 100644 index 0000000000..800d6c3305 --- /dev/null +++ b/backend/.sqlx/query-a2f047f9ca4b8a47c985fa092ba0d2dc54f7169af1c940b84345c477865ae82c.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT trigger_kind::text AS \"kind!\", COUNT(*)::BIGINT AS \"count!\"\n FROM capture_config GROUP BY trigger_kind", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "kind!", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null, + null + ] + }, + "hash": "a2f047f9ca4b8a47c985fa092ba0d2dc54f7169af1c940b84345c477865ae82c" +} diff --git a/backend/.sqlx/query-ae1973cf7dda23c1583521c7edd1a7e3beb695edaa00adf875cfdbb81ebe96dc.json b/backend/.sqlx/query-ae1973cf7dda23c1583521c7edd1a7e3beb695edaa00adf875cfdbb81ebe96dc.json new file mode 100644 index 0000000000..e1a5206335 --- /dev/null +++ b/backend/.sqlx/query-ae1973cf7dda23c1583521c7edd1a7e3beb695edaa00adf875cfdbb81ebe96dc.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings SET\n auto_invite = jsonb_set(\n jsonb_set(\n COALESCE(auto_invite, '{}'::jsonb),\n '{instance_groups}',\n (SELECT COALESCE(jsonb_agg(\n CASE WHEN elem #>> '{}' = $1 THEN to_jsonb($2::text) ELSE elem END), '[]'::jsonb)\n FROM jsonb_array_elements(COALESCE(auto_invite->'instance_groups', '[]'::jsonb)) elem)\n ),\n '{instance_groups_roles}',\n CASE WHEN COALESCE(auto_invite->'instance_groups_roles', '{}'::jsonb) ? $1\n THEN (COALESCE(auto_invite->'instance_groups_roles', '{}'::jsonb) - $1)\n || jsonb_build_object($2::text, auto_invite->'instance_groups_roles'->$1)\n ELSE COALESCE(auto_invite->'instance_groups_roles', '{}'::jsonb)\n END\n )\n WHERE auto_invite->'instance_groups' ? $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "ae1973cf7dda23c1583521c7edd1a7e3beb695edaa00adf875cfdbb81ebe96dc" +} diff --git a/backend/.sqlx/query-bbc2638aae4fb3556c8d876e4efd402c7b7c93ff9fd89364afd41470a92a5e2d.json b/backend/.sqlx/query-bbc2638aae4fb3556c8d876e4efd402c7b7c93ff9fd89364afd41470a92a5e2d.json new file mode 100644 index 0000000000..e59183f51a --- /dev/null +++ b/backend/.sqlx/query-bbc2638aae4fb3556c8d876e4efd402c7b7c93ff9fd89364afd41470a92a5e2d.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT name FROM instance_group WHERE name <> ALL($1)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "TextArray" + ] + }, + "nullable": [ + false + ] + }, + "hash": "bbc2638aae4fb3556c8d876e4efd402c7b7c93ff9fd89364afd41470a92a5e2d" +} diff --git a/backend/.sqlx/query-c748617e060bc41b5922df4b433f20b6971b993e0671b6ccb45db5ef028550bc.json b/backend/.sqlx/query-c748617e060bc41b5922df4b433f20b6971b993e0671b6ccb45db5ef028550bc.json new file mode 100644 index 0000000000..7f6fe7a81c --- /dev/null +++ b/backend/.sqlx/query-c748617e060bc41b5922df4b433f20b6971b993e0671b6ccb45db5ef028550bc.json @@ -0,0 +1,33 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, dedicated_workers, wm_version, vcpus, memory, job_isolation, native_mode) VALUES ($1, $2, COALESCE($3, 'NO IP'), $4, $5, $6, $7, $8, $9, $10, $11, $12) ON CONFLICT (worker)\n DO UPDATE set ping_at = now(), worker_instance = EXCLUDED.worker_instance, ip = COALESCE($3, worker_ping.ip), custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group, dedicated_worker = EXCLUDED.dedicated_worker, dedicated_workers = EXCLUDED.dedicated_workers, wm_version = EXCLUDED.wm_version, vcpus = COALESCE(EXCLUDED.vcpus, worker_ping.vcpus), memory = COALESCE(EXCLUDED.memory, worker_ping.memory), job_isolation = EXCLUDED.job_isolation, native_mode = EXCLUDED.native_mode, current_job_id = NULL, current_job_workspace_id = NULL\n RETURNING jobs_executed", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "jobs_executed", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Text", + "TextArray", + "Varchar", + "Varchar", + "TextArray", + "Varchar", + "Int8", + "Int8", + "Text", + "Bool" + ] + }, + "nullable": [ + false + ] + }, + "hash": "c748617e060bc41b5922df4b433f20b6971b993e0671b6ccb45db5ef028550bc" +} diff --git a/backend/.sqlx/query-d069ad741996e3ea992bcfb33e290ba5b87be1c047c189c9b8eb13da97e2085d.json b/backend/.sqlx/query-d069ad741996e3ea992bcfb33e290ba5b87be1c047c189c9b8eb13da97e2085d.json new file mode 100644 index 0000000000..7e667694b9 --- /dev/null +++ b/backend/.sqlx/query-d069ad741996e3ea992bcfb33e290ba5b87be1c047c189c9b8eb13da97e2085d.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT pg_advisory_xact_lock(hashtext('reconcile_workspace_instance_groups'), hashtext($1))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "pg_advisory_xact_lock", + "type_info": "Void" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "d069ad741996e3ea992bcfb33e290ba5b87be1c047c189c9b8eb13da97e2085d" +} diff --git a/backend/.sqlx/query-d1f701e81fc98933802356f292455da604367ef35893c2bb095bcd637567b114.json b/backend/.sqlx/query-d1f701e81fc98933802356f292455da604367ef35893c2bb095bcd637567b114.json new file mode 100644 index 0000000000..064e7c38aa --- /dev/null +++ b/backend/.sqlx/query-d1f701e81fc98933802356f292455da604367ef35893c2bb095bcd637567b114.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT email, igroup FROM email_to_igroup WHERE igroup = ANY($1)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "igroup", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "TextArray" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "d1f701e81fc98933802356f292455da604367ef35893c2bb095bcd637567b114" +} diff --git a/backend/.sqlx/query-dab323eda1fcaff0435e98d77e544ed5d63dd6023b1dab77d5f188b299f03b9d.json b/backend/.sqlx/query-dab323eda1fcaff0435e98d77e544ed5d63dd6023b1dab77d5f188b299f03b9d.json deleted file mode 100644 index ee4c5ff1af..0000000000 --- a/backend/.sqlx/query-dab323eda1fcaff0435e98d77e544ed5d63dd6023b1dab77d5f188b299f03b9d.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE workspace_settings SET\n auto_invite = jsonb_set(\n jsonb_set(\n COALESCE(auto_invite, '{}'::jsonb),\n '{instance_groups}',\n (SELECT COALESCE(jsonb_agg(elem), '[]'::jsonb) FROM jsonb_array_elements(COALESCE(auto_invite->'instance_groups', '[]'::jsonb)) elem WHERE elem #>> '{}' != $1)\n ),\n '{instance_groups_roles}',\n COALESCE(auto_invite->'instance_groups_roles', '{}'::jsonb) - $1\n )\n WHERE auto_invite->'instance_groups' IS NOT NULL AND auto_invite->'instance_groups' ? $1", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [] - }, - "hash": "dab323eda1fcaff0435e98d77e544ed5d63dd6023b1dab77d5f188b299f03b9d" -} diff --git a/backend/.sqlx/query-db9b48f91a2387e08a2eaa5bda344edf83bfcb8c8c330d524ed4eaafedbdbc0e.json b/backend/.sqlx/query-db9b48f91a2387e08a2eaa5bda344edf83bfcb8c8c330d524ed4eaafedbdbc0e.json new file mode 100644 index 0000000000..05c496b552 --- /dev/null +++ b/backend/.sqlx/query-db9b48f91a2387e08a2eaa5bda344edf83bfcb8c8c330d524ed4eaafedbdbc0e.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO global_settings (name, value) VALUES ($1, $2)\n ON CONFLICT (name) DO UPDATE SET value = EXCLUDED.value\n WHERE jsonb_typeof(global_settings.value) <> 'string'\n RETURNING value", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "value", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Jsonb" + ] + }, + "nullable": [ + false + ] + }, + "hash": "db9b48f91a2387e08a2eaa5bda344edf83bfcb8c8c330d524ed4eaafedbdbc0e" +} diff --git a/backend/.sqlx/query-e24252d48a1fcca73f20d62f37c9d7dc2071580be6d604979c95b3374cd4ad77.json b/backend/.sqlx/query-e24252d48a1fcca73f20d62f37c9d7dc2071580be6d604979c95b3374cd4ad77.json new file mode 100644 index 0000000000..d52ba8fb85 --- /dev/null +++ b/backend/.sqlx/query-e24252d48a1fcca73f20d62f37c9d7dc2071580be6d604979c95b3374cd4ad77.json @@ -0,0 +1,173 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH scanned AS (\n SELECT a.* FROM (\n SELECT value FROM flow\n WHERE archived = false AND pg_column_size(value) < $1\n LIMIT $2\n ) f,\n LATERAL (\n SELECT\n bool_or(m->'value'->>'type' = 'forloopflow') AS forloop,\n bool_or(m->'value'->>'type' = 'whileloopflow') AS whileloop,\n bool_or(m->'value'->>'type' = 'branchall') AS branchall,\n bool_or(m->'value'->>'type' = 'branchall'\n AND m->'value'->>'parallel' = 'false') AS branchall_seq,\n bool_or(m->'value'->>'type' = 'branchone') AS branchone,\n bool_or(m->'value'->>'type' = 'aiagent') AS aiagent,\n bool_or(m->'value'->>'type' = 'flow') AS subflow,\n bool_or(m->'value'->>'type' = 'identity') AS identity,\n bool_or(m->'value'->>'is_trigger' = 'true') AS trigger_step,\n bool_or(m->'value'->>'squash' = 'true') AS squash,\n bool_or(m->'value'->>'type' IN ('forloopflow', 'whileloopflow')\n AND m->'value'->>'parallel' = 'true') AS parallel_loop,\n bool_or(m->'value'->>'type' IN ('forloopflow', 'whileloopflow')\n AND m->'value'->>'skip_failures' = 'false') AS keep_failures,\n bool_or(m->'value' ? 'parallelism') AS parallelism,\n bool_or(m ? 'sleep') AS sleep,\n bool_or(m ? 'cache_ttl') AS cache,\n bool_or(m->'mock'->>'enabled' = 'true') AS mock,\n bool_or(m ? 'suspend') AS suspend,\n bool_or(m ? 'retry') AS retry,\n bool_or(m ? 'timeout') AS timeout,\n bool_or(m ? 'priority') AS priority,\n bool_or(m ? 'debouncing') AS debounce,\n bool_or(m ? 'delete_after_secs') AS lifetime,\n bool_or(m->>'continue_on_error' = 'true') AS continue_on_error,\n bool_or(m ? 'stop_after_if' OR m ? 'stop_after_all_iters_if') AS early_stop,\n bool_or(m ? 'skip_if') AS skip\n FROM jsonb_path_query(f.value, '$.**.modules[*]') m\n ) a\n )\n SELECT\n COUNT(*)::BIGINT AS \"flows_scanned!\",\n COUNT(*) FILTER (WHERE forloop)::BIGINT AS \"forloopflow!\",\n COUNT(*) FILTER (WHERE whileloop)::BIGINT AS \"whileloopflow!\",\n COUNT(*) FILTER (WHERE branchall)::BIGINT AS \"branchall!\",\n COUNT(*) FILTER (WHERE branchall_seq)::BIGINT AS \"branchall_sequential!\",\n COUNT(*) FILTER (WHERE branchone)::BIGINT AS \"branchone!\",\n COUNT(*) FILTER (WHERE aiagent)::BIGINT AS \"aiagent!\",\n COUNT(*) FILTER (WHERE subflow)::BIGINT AS \"subflow!\",\n COUNT(*) FILTER (WHERE identity)::BIGINT AS \"identity!\",\n COUNT(*) FILTER (WHERE trigger_step)::BIGINT AS \"trigger_step!\",\n COUNT(*) FILTER (WHERE squash)::BIGINT AS \"squash!\",\n COUNT(*) FILTER (WHERE parallel_loop)::BIGINT AS \"parallel_loop!\",\n COUNT(*) FILTER (WHERE keep_failures)::BIGINT AS \"keep_failures!\",\n COUNT(*) FILTER (WHERE parallelism)::BIGINT AS \"parallelism!\",\n COUNT(*) FILTER (WHERE sleep)::BIGINT AS \"sleep!\",\n COUNT(*) FILTER (WHERE cache)::BIGINT AS \"cache!\",\n COUNT(*) FILTER (WHERE mock)::BIGINT AS \"mock!\",\n COUNT(*) FILTER (WHERE suspend)::BIGINT AS \"suspend!\",\n COUNT(*) FILTER (WHERE retry)::BIGINT AS \"retry!\",\n COUNT(*) FILTER (WHERE timeout)::BIGINT AS \"timeout!\",\n COUNT(*) FILTER (WHERE priority)::BIGINT AS \"priority!\",\n COUNT(*) FILTER (WHERE debounce)::BIGINT AS \"debounce!\",\n COUNT(*) FILTER (WHERE lifetime)::BIGINT AS \"lifetime!\",\n COUNT(*) FILTER (WHERE continue_on_error)::BIGINT AS \"continue_on_error!\",\n COUNT(*) FILTER (WHERE early_stop)::BIGINT AS \"early_stop!\",\n COUNT(*) FILTER (WHERE skip)::BIGINT AS \"skip!\"\n FROM scanned\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "flows_scanned!", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "forloopflow!", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "whileloopflow!", + "type_info": "Int8" + }, + { + "ordinal": 3, + "name": "branchall!", + "type_info": "Int8" + }, + { + "ordinal": 4, + "name": "branchall_sequential!", + "type_info": "Int8" + }, + { + "ordinal": 5, + "name": "branchone!", + "type_info": "Int8" + }, + { + "ordinal": 6, + "name": "aiagent!", + "type_info": "Int8" + }, + { + "ordinal": 7, + "name": "subflow!", + "type_info": "Int8" + }, + { + "ordinal": 8, + "name": "identity!", + "type_info": "Int8" + }, + { + "ordinal": 9, + "name": "trigger_step!", + "type_info": "Int8" + }, + { + "ordinal": 10, + "name": "squash!", + "type_info": "Int8" + }, + { + "ordinal": 11, + "name": "parallel_loop!", + "type_info": "Int8" + }, + { + "ordinal": 12, + "name": "keep_failures!", + "type_info": "Int8" + }, + { + "ordinal": 13, + "name": "parallelism!", + "type_info": "Int8" + }, + { + "ordinal": 14, + "name": "sleep!", + "type_info": "Int8" + }, + { + "ordinal": 15, + "name": "cache!", + "type_info": "Int8" + }, + { + "ordinal": 16, + "name": "mock!", + "type_info": "Int8" + }, + { + "ordinal": 17, + "name": "suspend!", + "type_info": "Int8" + }, + { + "ordinal": 18, + "name": "retry!", + "type_info": "Int8" + }, + { + "ordinal": 19, + "name": "timeout!", + "type_info": "Int8" + }, + { + "ordinal": 20, + "name": "priority!", + "type_info": "Int8" + }, + { + "ordinal": 21, + "name": "debounce!", + "type_info": "Int8" + }, + { + "ordinal": 22, + "name": "lifetime!", + "type_info": "Int8" + }, + { + "ordinal": 23, + "name": "continue_on_error!", + "type_info": "Int8" + }, + { + "ordinal": 24, + "name": "early_stop!", + "type_info": "Int8" + }, + { + "ordinal": 25, + "name": "skip!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Int4", + "Int8" + ] + }, + "nullable": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ] + }, + "hash": "e24252d48a1fcca73f20d62f37c9d7dc2071580be6d604979c95b3374cd4ad77" +} diff --git a/backend/.sqlx/query-e242c733ca0accb5287e80c97abe3bbae0638612bd111b6b6196dd2808d4b6d0.json b/backend/.sqlx/query-e242c733ca0accb5287e80c97abe3bbae0638612bd111b6b6196dd2808d4b6d0.json new file mode 100644 index 0000000000..d28a19f599 --- /dev/null +++ b/backend/.sqlx/query-e242c733ca0accb5287e80c97abe3bbae0638612bd111b6b6196dd2808d4b6d0.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings SET\n auto_invite = jsonb_set(\n jsonb_set(\n COALESCE(auto_invite, '{}'::jsonb),\n '{instance_groups}',\n (SELECT COALESCE(jsonb_agg(elem), '[]'::jsonb)\n FROM jsonb_array_elements(COALESCE(auto_invite->'instance_groups', '[]'::jsonb)) elem\n WHERE elem #>> '{}' <> ALL($1))\n ),\n '{instance_groups_roles}',\n CASE WHEN jsonb_typeof(auto_invite->'instance_groups_roles') = 'object'\n THEN (auto_invite->'instance_groups_roles') - $1::text[]\n ELSE '{}'::jsonb\n END\n )\n WHERE auto_invite->'instance_groups' ?| $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "e242c733ca0accb5287e80c97abe3bbae0638612bd111b6b6196dd2808d4b6d0" +} diff --git a/backend/.sqlx/query-565db14b889f69dfbda5db400a33223fd20548fb106d2a5e5669c0e23ecaa0eb.json b/backend/.sqlx/query-e4e621724d830b318c06734683c8a48b0b692f7e390f2e95162d39ff3f6aeba7.json similarity index 64% rename from backend/.sqlx/query-565db14b889f69dfbda5db400a33223fd20548fb106d2a5e5669c0e23ecaa0eb.json rename to backend/.sqlx/query-e4e621724d830b318c06734683c8a48b0b692f7e390f2e95162d39ff3f6aeba7.json index af526daf8f..fee76e4be4 100644 --- a/backend/.sqlx/query-565db14b889f69dfbda5db400a33223fd20548fb106d2a5e5669c0e23ecaa0eb.json +++ b/backend/.sqlx/query-e4e621724d830b318c06734683c8a48b0b692f7e390f2e95162d39ff3f6aeba7.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT name FROM instance_group WHERE name = $1", + "query": "SELECT name FROM instance_group WHERE id = $1 FOR UPDATE", "describe": { "columns": [ { @@ -18,5 +18,5 @@ false ] }, - "hash": "565db14b889f69dfbda5db400a33223fd20548fb106d2a5e5669c0e23ecaa0eb" + "hash": "e4e621724d830b318c06734683c8a48b0b692f7e390f2e95162d39ff3f6aeba7" } diff --git a/backend/.sqlx/query-e52a80386b132e53a956458e2eb77d29bd6da4fc8511ed44f21d49f4c0965d36.json b/backend/.sqlx/query-e52a80386b132e53a956458e2eb77d29bd6da4fc8511ed44f21d49f4c0965d36.json new file mode 100644 index 0000000000..26a5b9f932 --- /dev/null +++ b/backend/.sqlx/query-e52a80386b132e53a956458e2eb77d29bd6da4fc8511ed44f21d49f4c0965d36.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO trigger_history\n (workspace_id, trigger_kind, path, operation, source, username, changes)\n VALUES ($1, $2, $3, $4, $5, $6, $7)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "e52a80386b132e53a956458e2eb77d29bd6da4fc8511ed44f21d49f4c0965d36" +} diff --git a/backend/.sqlx/query-e63e275a158040659c41ec8d1ef9107558b0003f495ba3f5fba63b77616d12c8.json b/backend/.sqlx/query-e63e275a158040659c41ec8d1ef9107558b0003f495ba3f5fba63b77616d12c8.json new file mode 100644 index 0000000000..d5a9d425da --- /dev/null +++ b/backend/.sqlx/query-e63e275a158040659c41ec8d1ef9107558b0003f495ba3f5fba63b77616d12c8.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT kind::text AS \"kind!\", COUNT(*)::BIGINT AS \"count!\"\n FROM script WHERE archived = false AND deleted = false GROUP BY kind", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "kind!", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null, + null + ] + }, + "hash": "e63e275a158040659c41ec8d1ef9107558b0003f495ba3f5fba63b77616d12c8" +} diff --git a/backend/.sqlx/query-ee182378d9a760c3593b483703ec1682488ca9e6e402ec41abaa6ae3ca263854.json b/backend/.sqlx/query-ee182378d9a760c3593b483703ec1682488ca9e6e402ec41abaa6ae3ca263854.json deleted file mode 100644 index ec170d17c8..0000000000 --- a/backend/.sqlx/query-ee182378d9a760c3593b483703ec1682488ca9e6e402ec41abaa6ae3ca263854.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT email_to_igroup.email\n FROM email_to_igroup\n INNER JOIN instance_group ON instance_group.name = email_to_igroup.igroup\n WHERE instance_group.name = $1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "email", - "type_info": "Varchar" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - false - ] - }, - "hash": "ee182378d9a760c3593b483703ec1682488ca9e6e402ec41abaa6ae3ca263854" -} diff --git a/backend/.sqlx/query-fcbbc3b697249c6ee0ca542ea42fadecddead568110dd531fbace439c2263e4f.json b/backend/.sqlx/query-fcbbc3b697249c6ee0ca542ea42fadecddead568110dd531fbace439c2263e4f.json new file mode 100644 index 0000000000..b039aab41c --- /dev/null +++ b/backend/.sqlx/query-fcbbc3b697249c6ee0ca542ea42fadecddead568110dd531fbace439c2263e4f.json @@ -0,0 +1,71 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, trigger_kind, path, operation, source, username, created_at, changes\n FROM trigger_history\n WHERE workspace_id = $1\n AND ($2::TEXT IS NULL OR trigger_kind = $2)\n AND ($3::TEXT IS NULL OR path = $3)\n AND ( $6\n OR path = ANY($7)\n OR EXISTS ( SELECT 1 FROM unnest($8::text[]) AS pfx\n WHERE path = pfx\n OR left(path, length(pfx) + 1) = pfx || '/' ) )\n ORDER BY id DESC\n LIMIT $4 OFFSET $5", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "trigger_kind", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "operation", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "source", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "username", + "type_info": "Varchar" + }, + { + "ordinal": 6, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 7, + "name": "changes", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Int8", + "Int8", + "Bool", + "TextArray", + "TextArray" + ] + }, + "nullable": [ + false, + false, + false, + false, + false, + true, + false, + true + ] + }, + "hash": "fcbbc3b697249c6ee0ca542ea42fadecddead568110dd531fbace439c2263e4f" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 3c965b2a9a..e740bcb3c2 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -15359,7 +15359,6 @@ dependencies = [ "argon2", "axum 0.8.9", "chrono", - "dashmap", "http 1.5.0", "hyper 1.11.0", "lazy_static", diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index ac0d037e67..441098c37b 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -71ef2cf2a5badd53d16d5cc945ab4ada1a639314 +1e7cd2fc8c2ee2b1c7f8589b9c917dc070f55e77 diff --git a/backend/migrations/20260813195023_preserve_orphaned_instance_group_members.down.sql b/backend/migrations/20260813195023_preserve_orphaned_instance_group_members.down.sql new file mode 100644 index 0000000000..c2eb12a15a --- /dev/null +++ b/backend/migrations/20260813195023_preserve_orphaned_instance_group_members.down.sql @@ -0,0 +1,9 @@ +-- Restore the instance_group source for members the up migration converted. The stripped +-- auto_invite references cannot be restored (the groups they named no longer exist). +UPDATE usr +SET added_via = jsonb_build_object( + 'source', 'instance_group', + 'group', added_via->>'migrated_from_instance_group' +) +WHERE added_via->>'source' = 'manual' + AND added_via ? 'migrated_from_instance_group'; diff --git a/backend/migrations/20260813195023_preserve_orphaned_instance_group_members.up.sql b/backend/migrations/20260813195023_preserve_orphaned_instance_group_members.up.sql new file mode 100644 index 0000000000..581401b854 --- /dev/null +++ b/backend/migrations/20260813195023_preserve_orphaned_instance_group_members.up.sql @@ -0,0 +1,54 @@ +-- Strip auto_invite references to groups that no longer exist, so a later group created +-- with the same name cannot silently re-acquire the mapping. +UPDATE workspace_settings +SET auto_invite = jsonb_set( + jsonb_set( + auto_invite, + '{instance_groups}', + COALESCE( + (SELECT jsonb_agg(elem) + FROM jsonb_array_elements(auto_invite->'instance_groups') elem + WHERE EXISTS (SELECT 1 FROM instance_group ig WHERE ig.name = elem #>> '{}')), + '[]'::jsonb + ) + ), + '{instance_groups_roles}', + CASE WHEN jsonb_typeof(auto_invite->'instance_groups_roles') = 'object' + THEN (SELECT COALESCE(jsonb_object_agg(key, value), '{}'::jsonb) + FROM jsonb_each(auto_invite->'instance_groups_roles') + WHERE EXISTS (SELECT 1 FROM instance_group ig WHERE ig.name = key)) + ELSE '{}'::jsonb + END +) +WHERE jsonb_typeof(auto_invite->'instance_groups') = 'array' + AND EXISTS ( + SELECT 1 FROM jsonb_array_elements(auto_invite->'instance_groups') elem + WHERE NOT EXISTS (SELECT 1 FROM instance_group ig WHERE ig.name = elem #>> '{}') + ); + +-- Workspace members whose instance-group grant can no longer be re-derived become manual +-- members. Group deletion, overwrite imports and some SCIM paths used to mutate groups +-- without carrying the change through to workspace membership, leaving members whose +-- granting group was deleted — or who were dropped from a group that still exists. Under +-- state-based reconciliation those members belong to zero configured groups, so the first +-- reconcile touching their workspace would otherwise remove them and destroy their drafts, +-- favorites, tokens and permissions. The original group name is kept under +-- 'migrated_from_instance_group' so admins can identify and prune them deliberately. +UPDATE usr +SET added_via = jsonb_build_object( + 'source', 'manual', + 'migrated_from_instance_group', added_via->>'group' +) +WHERE added_via->>'source' = 'instance_group' + AND NOT EXISTS ( + SELECT 1 + FROM workspace_settings ws + JOIN LATERAL jsonb_array_elements_text( + CASE WHEN jsonb_typeof(ws.auto_invite->'instance_groups') = 'array' + THEN ws.auto_invite->'instance_groups' + ELSE '[]'::jsonb + END + ) g ON true + JOIN email_to_igroup e ON e.igroup = g.value AND e.email = usr.email + WHERE ws.workspace_id = usr.workspace_id + ); diff --git a/backend/migrations/20260814090221_trigger_history.down.sql b/backend/migrations/20260814090221_trigger_history.down.sql new file mode 100644 index 0000000000..c4aad347b0 --- /dev/null +++ b/backend/migrations/20260814090221_trigger_history.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS trigger_history; diff --git a/backend/migrations/20260814090221_trigger_history.up.sql b/backend/migrations/20260814090221_trigger_history.up.sql new file mode 100644 index 0000000000..1eb4f778a2 --- /dev/null +++ b/backend/migrations/20260814090221_trigger_history.up.sql @@ -0,0 +1,65 @@ +-- Append-only record of every schedule/trigger mutation: who, what changed, and +-- from which kind of client. +CREATE TABLE IF NOT EXISTS trigger_history ( + id BIGSERIAL PRIMARY KEY, + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE, + -- 'schedule' or a trigger's TRIGGER_TYPE ('http', 'kafka', ...). Not the + -- TRIGGER_KIND enum: that one is capture-oriented and misses 'schedule'. + trigger_kind VARCHAR(50) NOT NULL, + path VARCHAR(255) NOT NULL, + -- 'create' | 'update' | 'delete' | 'enable' | 'disable' | 'suspend' + operation VARCHAR(20) NOT NULL, + -- 'ui' | 'cli' | 'api' | 'worker' + source VARCHAR(20) NOT NULL, + -- NULL when the server acted on its own (worker auto-disable). + username VARCHAR(255), + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), + -- {field: {old, new}} for the fields that actually changed. `old` is + -- absent where it is not known: a create, and the workspace-wide handler + -- override that rewrites every schedule without reading them first. NULL + -- when the operation carries no field-level diff at all (delete). + changes JSONB +); + +CREATE INDEX IF NOT EXISTS idx_trigger_history_workspace_kind_path + ON trigger_history(workspace_id, trigger_kind, path, id DESC); + +CREATE INDEX IF NOT EXISTS idx_trigger_history_workspace_id + ON trigger_history(workspace_id, id DESC); + +GRANT ALL ON TABLE trigger_history TO windmill_user; +GRANT ALL ON TABLE trigger_history TO windmill_admin; +GRANT ALL ON SEQUENCE trigger_history_id_seq TO windmill_user; +GRANT ALL ON SEQUENCE trigger_history_id_seq TO windmill_admin; + +ALTER TABLE trigger_history ENABLE ROW LEVEL SECURITY; + +CREATE POLICY admin_all ON trigger_history FOR ALL TO windmill_admin USING (true) WITH CHECK (true); + +-- Every mutating trigger route writes through the RLS pool, so windmill_user +-- must be able to append. +CREATE POLICY allow_insert ON trigger_history FOR INSERT TO windmill_user WITH CHECK (true); + +-- Reads mirror the path half of the live trigger's own policies: a row can +-- quote a schedule's `args`, so it must not be readable by anyone who could not +-- read the trigger it describes. Deliberately narrower than the live row on one +-- point — the `extra_perms` grants have no counterpart here, since the history +-- does not carry the row's ACL and must survive its deletion. Narrower is the +-- safe direction. +CREATE POLICY see_own ON trigger_history FOR SELECT TO windmill_user +USING ( + SPLIT_PART(path::text, '/', 1) = 'u' + AND SPLIT_PART(path::text, '/', 2) = current_setting('session.user') +); + +CREATE POLICY see_member ON trigger_history FOR SELECT TO windmill_user +USING ( + SPLIT_PART(path::text, '/', 1) = 'g' + AND SPLIT_PART(path::text, '/', 2) = ANY(regexp_split_to_array(current_setting('session.groups'), ',')) +); + +CREATE POLICY see_folder_extra_perms_user ON trigger_history FOR SELECT TO windmill_user +USING ( + SPLIT_PART(path::text, '/', 1) = 'f' + AND SPLIT_PART(path::text, '/', 2) = ANY(regexp_split_to_array(current_setting('session.folders_read'), ',')) +); diff --git a/backend/oauth_connect.json b/backend/oauth_connect.json index 9e74822d98..cec49861cd 100644 --- a/backend/oauth_connect.json +++ b/backend/oauth_connect.json @@ -159,7 +159,8 @@ "sage_intacct": { "auth_url": "https://api.intacct.com/ia/api/v1/oauth2/authorize", "token_url": "https://api.intacct.com/ia/api/v1/oauth2/token", - "scopes": ["offline_access"] + "scopes": ["offline_access"], + "req_body_auth": true }, "spotify": { "auth_url": "https://accounts.spotify.com/authorize", diff --git a/backend/src/main.rs b/backend/src/main.rs index 43a92ce4bf..6383bfd78e 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -1691,7 +1691,8 @@ async fn process_notify_event( "restart_worker_group" => { if worker_mode && payload == *WORKER_GROUP { tracing::info!("Restart requested for worker group '{payload}'"); - spawn_graceful_killpill(tx, db, 30, "worker group restart requested").await; + spawn_graceful_killpill(tx, db, 30, "worker group restart requested", server_mode) + .await; } } "notify_webhook_change" => { @@ -1997,8 +1998,14 @@ async fn process_notify_event( reload_otel_tracing_proxy_setting(conn).await; if worker_mode { tracing::info!("OTEL tracing proxy setting changed, restarting worker"); - spawn_graceful_killpill(tx, db, 30, "OTEL tracing proxy setting change") - .await; + spawn_graceful_killpill( + tx, + db, + 30, + "OTEL tracing proxy setting change", + server_mode, + ) + .await; } } REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING => { @@ -2009,12 +2016,20 @@ async fn process_notify_event( } EXPOSE_METRICS_SETTING => { tracing::info!("Metrics setting changed, restarting"); - spawn_graceful_killpill(tx, db, 30, "metrics setting change").await; + spawn_graceful_killpill(tx, db, 30, "metrics setting change", server_mode) + .await; } EMAIL_DOMAIN_SETTING => { tracing::info!("Email domain setting changed"); if server_mode { - spawn_graceful_killpill(tx, db, 30, "email domain setting change").await; + spawn_graceful_killpill( + tx, + db, + 30, + "email domain setting change", + server_mode, + ) + .await; } } EXPOSE_DEBUG_METRICS_SETTING => { @@ -2050,19 +2065,26 @@ async fn process_notify_event( } OTEL_SETTING => { tracing::info!("OTEL setting changed, restarting"); - spawn_graceful_killpill(tx, db, 30, "OTEL setting change").await; + spawn_graceful_killpill(tx, db, 30, "OTEL setting change", server_mode).await; } REQUEST_SIZE_LIMIT_SETTING => { if server_mode { tracing::info!("Request limit size change detected, killing server expecting to be restarted"); - spawn_graceful_killpill(tx, db, 30, "request size limit change").await; + spawn_graceful_killpill( + tx, + db, + 30, + "request size limit change", + server_mode, + ) + .await; } } SAML_METADATA_SETTING => { tracing::info!( "SAML metadata change detected, killing server expecting to be restarted" ); - spawn_graceful_killpill(tx, db, 30, "SAML metadata change").await; + spawn_graceful_killpill(tx, db, 30, "SAML metadata change", server_mode).await; } HUB_BASE_URL_SETTING => { if let Err(e) = reload_hub_base_url_setting(conn, server_mode).await { @@ -2183,12 +2205,7 @@ pub async fn run_workers( // #[cfg(tokio_unstable)] // let monitor = tokio_metrics::TaskMonitor::new(); - let ip = windmill_common::external_ip::get_ip() - .await - .unwrap_or_else(|e| { - tracing::warn!(error = e.to_string(), "failed to get external IP"); - "unretrievable IP".to_string() - }); + windmill_common::external_ip::resolve_ip_in_background(); let mut handles = Vec::with_capacity(num_workers as usize); @@ -2232,7 +2249,6 @@ pub async fn run_workers( let conn1 = wk_conf.conn.clone(); let worker_name = wk_conf.worker_name.clone(); WORKERS_NAMES.write().await.push(worker_name.clone()); - let ip = ip.clone(); let rx = killpill_rxs.pop().unwrap(); let tx = tx.clone(); let base_internal_url = base_internal_url.clone(); @@ -2249,7 +2265,6 @@ pub async fn run_workers( worker_name, i as u64, num_workers as u32, - &ip, rx, tx, &base_internal_url, @@ -2286,16 +2301,24 @@ pub async fn run_workers( /// then the sleep+kill is spawned in the background so the notification handler is not blocked. /// /// Falls back to drain-only delay if DB coordination fails. +/// +/// Only `server_mode` processes coordinate, on the strength of the worker case: a worker +/// group restarting costs queue latency rather than lost work, `v2_job_queue` being durable. +/// Were workers to take part, one could claim the `is_first` slot and leave every server +/// holding its shutdown open for a peer that serves no API traffic. async fn spawn_graceful_killpill( tx: &KillpillSender, db: &Pool, safety_margin_secs: u64, context: &str, + server_mode: bool, ) { // Minimum delay before any restart to let in-flight requests drain const DRAIN_DELAY_SECS: u64 = 3; - let (delay, is_first) = + let (delay, is_first) = if !server_mode { + (DRAIN_DELAY_SECS, true) + } else { match coordinate_restart_delay(db, safety_margin_secs, DRAIN_DELAY_SECS).await { Ok(r) => r, Err(e) => { @@ -2305,7 +2328,8 @@ async fn spawn_graceful_killpill( ); (DRAIN_DELAY_SECS, true) } - }; + } + }; tracing::info!( "Scheduling {context} graceful shutdown in {delay}s (first_to_restart={is_first})" diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index a9f747cfe5..ab40597c9d 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -12,7 +12,7 @@ use std::{ }; use chrono::{DateTime, NaiveDateTime, Utc}; -use futures::{stream::FuturesUnordered, StreamExt}; +use futures::{future::BoxFuture, stream::FuturesUnordered, StreamExt}; use serde::{de::DeserializeOwned, Deserialize}; use sqlx::{Pool, Postgres}; use tokio::{ @@ -37,7 +37,6 @@ use windmill_common::ee_oss::low_disk_alerts; #[cfg(feature = "enterprise")] use windmill_common::ee_oss::{jobs_waiting_alerts, worker_groups_alerts}; -#[cfg(feature = "oauth2")] use windmill_common::global_settings::OAUTH_SETTING; use windmill_common::otel_oss::{ otel_incr_zombie_delete_count, otel_incr_zombie_restart_count, otel_set_db_pool, @@ -52,13 +51,14 @@ use windmill_common::{ error, flow_status::{FlowStatus, FlowStatusModule}, global_settings::{ - AUDIT_LOG_RETENTION_DAYS_SETTING, BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, - BUN_INSTALL_MIN_RELEASE_AGE_SETTING, CONCURRENCY_KEY_MAX_QUEUED_SETTING, - CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING, CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING, - CRITICAL_ALERT_MUTE_UI_SETTING, CRITICAL_ERROR_CHANNELS_SETTING, - DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, - DISABLE_PASSWORD_LOGIN, DISABLE_PASSWORD_LOGIN_SETTING, EXPOSE_DEBUG_METRICS_SETTING, - EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, + get_or_create_jwt_secret, load_value_from_global_settings, + AUDIT_LOG_RETENTION_DAYS_SETTING, BASE_URL_SETTING, + BUNFIG_INSTALL_SCOPES_SETTING, BUN_INSTALL_MIN_RELEASE_AGE_SETTING, + CONCURRENCY_KEY_MAX_QUEUED_SETTING, CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING, + CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING, + CRITICAL_ERROR_CHANNELS_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING, + DEFAULT_TAGS_WORKSPACES_SETTING, DISABLE_PASSWORD_LOGIN, DISABLE_PASSWORD_LOGIN_SETTING, + EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX_SETTING, HUB_API_SECRET_SETTING, HUB_BASE_URL_SETTING, INSTANCE_PYTHON_VERSION_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JOB_ISOLATION_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, @@ -70,8 +70,8 @@ use windmill_common::{ RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SANDBOX_IMAGE_CACHE_MAX_MB_SETTING, SANDBOX_IMAGE_DEFAULT_REGISTRY_SETTING, SANDBOX_IMAGE_MAX_SIZE_MB_SETTING, SANDBOX_IMAGE_PULL_POLICY_SETTING, SANDBOX_REGISTRY_AUTH_SETTING, SCIM_TOKEN_SETTING, - STORE_AUDIT_LOGS_S3_SETTING, TIMEOUT_WAIT_RESULT_SETTING, UV_EXCLUDE_NEWER_SETTING, - UV_INDEX_STRATEGY_SETTING, UV_PYTHON_INSTALL_MIRROR_SETTING, + SMTP_SETTING, STORE_AUDIT_LOGS_S3_SETTING, TIMEOUT_WAIT_RESULT_SETTING, + UV_EXCLUDE_NEWER_SETTING, UV_INDEX_STRATEGY_SETTING, UV_PYTHON_INSTALL_MIRROR_SETTING, WORKSPACE_FAIRNESS_DURATION_SECS_SETTING, WORKSPACE_FAIRNESS_ENABLED_SETTING, WORKSPACE_FAIRNESS_MAX_PERCENT_SETTING, WORKSPACE_FAIRNESS_MIN_TOTAL_SETTING, WORKSPACE_MAX_QUEUED_JOBS_SETTING, @@ -83,11 +83,11 @@ use windmill_common::{ server::load_smtp_config, tracing_init::JSON_FMT, users::truncate_token, - utils::{empty_as_none, now_from_db, rd_string, report_critical_error, Mode, HUB_API_SECRET}, + utils::{empty_as_none, now_from_db, report_critical_error, Mode, HUB_API_SECRET}, worker::{ load_env_vars, load_init_bash_from_env, load_periodic_bash_script_from_env, load_periodic_bash_script_interval_from_env, load_whitelist_env_vars_from_env, - load_worker_config, reload_custom_tags_setting, store_pull_query, + load_worker_config, store_pull_query, store_suspended_pull_query, Connection, WorkerConfig, CLOUD_HOSTED, CONCURRENCY_KEY_MAX_QUEUED, CONCURRENCY_KEY_MAX_QUEUED_DEFAULT, DEFAULT_TAGS_PER_WORKSPACE, DEFAULT_TAGS_WORKSPACES, FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX, INDEXER_CONFIG, @@ -225,6 +225,11 @@ lazy_static::lazy_static! { .unwrap_or(20); } +/// Load every setting this process cares about, at startup and on every full-reload tick. +/// +/// Reads are declared into a [`SettingsPass`] rather than issued one at a time, so the dozens +/// this pass makes cost a single fetch. See [`SettingsPass`] for what declaring buys and what +/// it requires of the order things are declared in. pub async fn initial_load( conn: &Connection, tx: KillpillSender, @@ -232,102 +237,134 @@ pub async fn initial_load( server_mode: bool, #[cfg(feature = "parquet")] disable_s3_store: bool, ) { - if let Err(e) = reload_base_url_setting(&conn).await { - tracing::error!("Error loading base url: {:?}", e) - } + let mut pass = SettingsPass::new(); + + pass.settings( + &[OAUTH_SETTING, BASE_URL_SETTING], + false, + move |mut v| async move { + if let Err(e) = apply_base_url_setting( + conn, + v.remove(OAUTH_SETTING).flatten(), + v.remove(BASE_URL_SETTING).flatten(), + ) + .await + { + tracing::error!("Error loading base url: {:?}", e) + } + }, + ); + + pass.setting(CRITICAL_ERROR_CHANNELS_SETTING, false, |v| async move { + apply_critical_error_channels_setting(v) + }); + + pass.setting(EXPOSE_METRICS_SETTING, true, |v| async move { + apply_metrics_enabled(v) + }); + pass.setting(EXPOSE_DEBUG_METRICS_SETTING, true, |v| async move { + apply_metrics_debug_enabled(v) + }); + pass.setting(CRITICAL_ALERT_MUTE_UI_SETTING, true, |v| async move { + apply_critical_alert_mute_ui_setting(v) + }); + pass.setting( + CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING, + true, + |v| async move { apply_critical_alerts_on_token_expiry_setting(v) }, + ); if let Some(db) = conn.as_sql() { - if let Err(e) = reload_critical_error_channels_setting(&db).await { - tracing::error!("Could loading critical error emails setting: {:?}", e); - } - } - - if let Err(e) = load_metrics_enabled(conn).await { - tracing::error!("Error loading expose metrics: {e:#}"); - } - - if let Err(e) = load_metrics_debug_enabled(conn).await { - tracing::error!("Error loading expose debug metrics: {e:#}"); - } - - if let Err(e) = reload_critical_alert_mute_ui_setting(conn).await { - tracing::error!("Error loading critical alert mute ui setting: {e:#}"); - } - - if let Err(e) = reload_critical_alerts_on_token_expiry_setting(conn).await { - tracing::error!("Error loading critical alerts on token expiry setting: {e:#}"); - } - - if let Some(db) = conn.as_sql() { - if let Err(e) = load_tag_per_workspace_enabled(db).await { - tracing::error!("Error loading default tag per workpsace: {e:#}"); - } - - if let Err(e) = load_tag_per_workspace_workspaces(db).await { - tracing::error!("Error loading default tag per workpsace workspaces: {e:#}"); - } - - if let Err(e) = load_fork_workspace_tag_append_fork_suffix(db).await { - tracing::error!("Error loading fork workspace tag append fork suffix: {e:#}"); - } - - if let Err(e) = load_preview_tags_override(db).await { - tracing::error!("Error loading preview tags override: {e:#}"); - } + pass.setting(DEFAULT_TAGS_PER_WORKSPACE_SETTING, false, |v| async move { + apply_tag_per_workspace_enabled(v) + }); + pass.setting(DEFAULT_TAGS_WORKSPACES_SETTING, false, |v| async move { + apply_tag_per_workspace_workspaces(v) + }); + pass.setting( + FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX_SETTING, + false, + |v| async move { apply_fork_workspace_tag_append_fork_suffix(v) }, + ); + pass.setting(PREVIEW_TAGS_OVERRIDE_SETTING, false, |v| async move { + apply_preview_tags_override(v) + }); // Load per-workspace retention overrides before the first cleanup tick so a fresh server // never sweeps globally without honoring configured longer-retention workspaces. - if let Err(e) = load_retention_period_overrides(db).await { - tracing::error!("Error loading per-workspace retention overrides: {e:#}"); - } + pass.action(async move { + if let Err(e) = load_retention_period_overrides(db).await { + tracing::error!("Error loading per-workspace retention overrides: {e:#}"); + } + }); - // Workspace fairness (cloud-only). Load the percentage/duration/min knobs - // *before* the enabled flag so that `load_workspace_fairness_enabled` reads - // current values when re-storing the pull queries. - if let Err(e) = load_workspace_fairness_max_percent(db).await { - tracing::error!("Error loading workspace fairness max percent: {e:#}"); - } - if let Err(e) = load_workspace_fairness_duration_secs(db).await { - tracing::error!("Error loading workspace fairness duration secs: {e:#}"); - } - if let Err(e) = load_workspace_fairness_min_total(db).await { - tracing::error!("Error loading workspace fairness min total: {e:#}"); - } - if let Err(e) = load_workspace_fairness_enabled(db).await { - tracing::error!("Error loading workspace fairness enabled: {e:#}"); - } + // Workspace fairness (cloud-only). The percentage/duration/min knobs apply + // *before* the enabled flag so that `apply_workspace_fairness_enabled` reads + // current values when re-storing the pull queries, which declaration order gives us. + pass.setting( + WORKSPACE_FAIRNESS_MAX_PERCENT_SETTING, + false, + |v| async move { apply_workspace_fairness_max_percent(v) }, + ); + pass.setting( + WORKSPACE_FAIRNESS_DURATION_SECS_SETTING, + false, + |v| async move { apply_workspace_fairness_duration_secs(v) }, + ); + pass.setting( + WORKSPACE_FAIRNESS_MIN_TOTAL_SETTING, + false, + |v| async move { apply_workspace_fairness_min_total(v) }, + ); + pass.setting(WORKSPACE_FAIRNESS_ENABLED_SETTING, false, |v| { + apply_workspace_fairness_enabled(v) + }); - // Only the cloud reads this cap, so don't spend a query loading it anywhere else. + // Only the cloud reads these caps, so don't ask for them anywhere else. if *CLOUD_HOSTED { - if let Err(e) = load_concurrency_key_max_queued(db).await { - tracing::error!("Error loading concurrency key max queued: {e:#}"); - } - if let Err(e) = load_workspace_max_queued_jobs(db).await { - tracing::error!("Error loading workspace max queued jobs: {e:#}"); - } + pass.setting(CONCURRENCY_KEY_MAX_QUEUED_SETTING, false, |v| async move { + apply_concurrency_key_max_queued(v) + }); + pass.setting(WORKSPACE_MAX_QUEUED_JOBS_SETTING, false, |v| async move { + apply_workspace_max_queued_jobs(v) + }); } } if server_mode { if let Some(db) = conn.as_sql() { - load_require_preexisting_user(db).await; - load_disable_password_login(db).await; - if let Err(e) = reload_critical_alerts_on_db_oversize(db).await { - tracing::error!( - "Error reloading critical alerts on db oversize setting: {:?}", - e - ) - } - windmill_common::min_version::store_min_keep_alive_version(db).await; - reload_instance_events_webhook_setting(db).await; + pass.setting( + REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, + false, + |v| async move { apply_require_preexisting_user(v) }, + ); + pass.setting(DISABLE_PASSWORD_LOGIN_SETTING, false, |v| async move { + apply_disable_password_login(v) + }); + pass.action(async move { + if let Err(e) = reload_critical_alerts_on_db_oversize(db).await { + tracing::error!( + "Error reloading critical alerts on db oversize setting: {:?}", + e + ) + } + }); + pass.action(windmill_common::min_version::store_min_keep_alive_version(db)); + pass.setting( + windmill_common::global_settings::INSTANCE_EVENTS_WEBHOOK_SETTING, + false, + |v| async move { apply_instance_events_webhook_setting(v) }, + ); } } if worker_mode { - load_keep_job_dir(conn).await; + pass.setting(KEEP_JOB_DIR_SETTING, true, |v| async move { + apply_keep_job_dir(v) + }); match conn { Connection::Sql(db) => { - reload_worker_config(&db, tx, false).await; + pass.action(reload_worker_config(&db, tx, false)); } Connection::Http(_) => { // TODO: reload worker config from http @@ -359,61 +396,119 @@ pub async fn initial_load( } } - if let Err(e) = reload_hub_base_url_setting(conn, server_mode).await { - tracing::error!("Error reloading hub base url: {:?}", e) - } + pass.setting(HUB_BASE_URL_SETTING, true, move |v| async move { + if let Err(e) = apply_hub_base_url_setting(conn, server_mode, v).await { + tracing::error!("Error reloading hub base url: {:?}", e) + } + }); if let Some(db) = conn.as_sql() { - if let Err(e) = reload_jwt_secret_setting(db).await { - tracing::error!("Could not reload jwt secret setting: {:?}", e); - } + pass.setting(JWT_SECRET_SETTING, false, move |v| async move { + if let Err(e) = apply_jwt_secret_setting(db, v).await { + tracing::error!("Could not reload jwt secret setting: {:?}", e); + } + }); - if let Err(e) = reload_custom_tags_setting(db).await { - tracing::error!("Error reloading custom tags: {:?}", e) - } - - if let Err(e) = reload_app_workspaced_route_setting(db).await { - tracing::error!("Error reloading app workspaced route: {:?}", e) - } - - if let Err(e) = reload_http_route_workspaced_route_setting(db).await { - tracing::error!("Error reloading http route workspaced route: {:?}", e) - } + pass.setting(CUSTOM_TAGS_SETTING, false, |v| async move { + windmill_common::worker::apply_custom_tags_setting(v) + }); + pass.setting(APP_WORKSPACED_ROUTE_SETTING, false, |v| async move { + apply_app_workspaced_route_setting(v) + }); + pass.setting( + HTTP_ROUTE_WORKSPACED_ROUTE_SETTING, + false, + move |v| async move { + if let Err(e) = apply_http_route_workspaced_route_setting(db, v).await { + tracing::error!("Error reloading http route workspaced route: {:?}", e) + } + }, + ); } + // A step rather than a plain await: an AWS OIDC store mints its first token against an + // issuer built from `BASE_URL` (`oidc_ee.rs`), and with `OTEL_ENVIRONMENT` set nothing + // loads that before this pass does, so running ahead of the applier would sign with the + // unset default and fall back to the 10s retry. #[cfg(feature = "parquet")] if !disable_s3_store { if let Some(db) = conn.as_sql() { let db2 = db.clone(); - match reload_object_store_setting(db).await { - ObjectStoreReload::Later => { - tokio::spawn(async move { - tokio::time::sleep(Duration::from_secs(10)).await; - match reload_object_store_setting(&db2).await { - ObjectStoreReload::Later => { - tracing::error!("Giving up on loading object store setting"); + pass.action(async move { + match reload_object_store_setting(db).await { + ObjectStoreReload::Later => { + tokio::spawn(async move { + tokio::time::sleep(Duration::from_secs(10)).await; + match reload_object_store_setting(&db2).await { + ObjectStoreReload::Later => { + tracing::error!("Giving up on loading object store setting"); + } + ObjectStoreReload::Never => { + tracing::info!("Object store setting successfully loaded"); + } } - ObjectStoreReload::Never => { - tracing::info!("Object store setting successfully loaded"); - } - } - }); + }); + } + ObjectStoreReload::Never => (), } - ObjectStoreReload::Never => (), - } + }); } } if let Some(db) = conn.as_sql() { - reload_smtp_config(db).await; + let _ = db; + pass.setting(SMTP_SETTING, false, |v| async move { + tracing::info!("Reloading smtp config..."); + SMTP_CONFIG.store(std::sync::Arc::new( + windmill_common::server::parse_smtp_config(v), + )); + }); } - reload_hub_api_secret_setting(&conn).await; + pass.option_setting_with( + HUB_API_SECRET_SETTING, + "HUB_API_SECRET", + |v: Option| async move { HUB_API_SECRET.store(std::sync::Arc::new(v)) }, + ); if server_mode { - reload_retention_period_setting(&conn).await; - reload_audit_log_retention_days_setting(&conn).await; - reload_store_audit_logs_s3_setting(&conn).await; + pass.setting(RETENTION_PERIOD_SECS_SETTING, true, |v| async move { + JOB_RETENTION_SECS.store( + parse_setting_value::( + v, + RETENTION_PERIOD_SECS_SETTING, + "JOB_RETENTION_SECS", + 60 * 60 * 24 * 30, + |x| x, + ), + Ordering::Relaxed, + ) + }); + pass.setting(AUDIT_LOG_RETENTION_DAYS_SETTING, true, |v| async move { + AUDIT_LOG_RETENTION_DAYS.store( + // 0 means use default: 365 for EE, 14 for CE + parse_setting_value::( + v, + AUDIT_LOG_RETENTION_DAYS_SETTING, + "AUDIT_LOG_RETENTION_DAYS", + 0, + |x| x, + ), + Ordering::Relaxed, + ) + }); + pass.setting(STORE_AUDIT_LOGS_S3_SETTING, true, |v| async move { + STORE_AUDIT_LOGS_S3.store( + parse_setting_value::( + v, + STORE_AUDIT_LOGS_S3_SETTING, + "STORE_AUDIT_LOGS_S3", + false, + |x| x, + ), + Ordering::Relaxed, + ) + }); // Env-var enable has no settings-row xmin and no runtime enable event; // anchor the export cursor at startup so rows committed before the // first export tick are not skipped (no-op when a settings row exists @@ -421,66 +516,177 @@ pub async fn initial_load( // Enterprise feature; the core logic lives in `crate::ee` (OSS gets a // no-op), gated here on a valid Enterprise license. #[cfg(feature = "parquet")] - if STORE_AUDIT_LOGS_S3.load(std::sync::atomic::Ordering::Relaxed) - && matches!( - windmill_common::ee_oss::get_license_plan().await, - windmill_common::ee_oss::LicensePlan::Enterprise - ) - { - if let Some(db) = conn.as_sql() { - crate::ee_oss::anchor_audit_logs_s3_checkpoint_env_var(&db).await; - } - } - reload_request_size(&conn).await; - reload_saml_metadata_setting(&conn).await; - reload_scim_token_setting(&conn).await; - - // Ensure audit partitions exist before any requests arrive if let Some(db) = conn.as_sql() { - manage_audit_partitions(&db, audit_log_retention_days().await).await; + pass.action(async move { + if STORE_AUDIT_LOGS_S3.load(std::sync::atomic::Ordering::Relaxed) + && matches!( + windmill_common::ee_oss::get_license_plan().await, + windmill_common::ee_oss::LicensePlan::Enterprise + ) + { + crate::ee_oss::anchor_audit_logs_s3_checkpoint_env_var(&db).await; + } + }); + } + pass.required_setting( + REQUEST_SIZE_LIMIT_SETTING, + "REQUEST_SIZE_LIMIT", + DEFAULT_BODY_LIMIT, + REQUEST_SIZE_LIMIT.clone(), + |x| x.mul(1024 * 1024), + ); + pass.option_setting( + SAML_METADATA_SETTING, + "SAML_METADATA", + SAML_METADATA.clone(), + ); + pass.option_setting(SCIM_TOKEN_SETTING, "SCIM_TOKEN", SCIM_TOKEN.clone()); + + // Ensure audit partitions exist before any requests arrive. A step rather than a + // plain await: it drops partitions past `audit_log_retention_days`, so running it + // before that setting is applied would sweep with the compile-time default. + if let Some(db) = conn.as_sql() { + pass.action(async move { + manage_audit_partitions(&db, audit_log_retention_days().await).await + }); } } if worker_mode { - reload_job_default_timeout_setting(&conn).await; - reload_job_isolation_setting(&conn).await; - reload_nsjail_tmpfs_size_setting(&conn).await; - reload_nsjail_tmp_backing_setting(&conn).await; - reload_sandbox_image_max_size_setting(&conn).await; - reload_sandbox_image_cache_max_setting(&conn).await; - reload_sandbox_image_pull_policy_setting(&conn).await; - reload_sandbox_image_default_registry_setting(&conn).await; - reload_sandbox_registry_auth_setting(&conn).await; - reload_extra_pip_index_url_setting(&conn).await; - reload_pip_index_url_setting(&conn).await; - reload_uv_index_strategy_setting(&conn).await; - reload_uv_exclude_newer_setting(&conn).await; - reload_uv_python_install_mirror_setting(&conn).await; - reload_bun_install_min_release_age_setting(&conn).await; - reload_npm_config_registry_setting(&conn).await; - reload_bunfig_install_scopes_setting(&conn).await; - reload_npmrc_setting(&conn).await; - reload_instance_python_version_setting(&conn).await; - reload_nuget_config_setting(&conn).await; - reload_powershell_repo_url_setting(&conn).await; - reload_powershell_repo_pat_setting(&conn).await; - reload_maven_repos_setting(&conn).await; - reload_maven_settings_xml_setting(&conn).await; - reload_no_default_maven_setting(&conn).await; - reload_ruby_repos_setting(&conn).await; - reload_cargo_registries_setting(&conn).await; - reload_workspace_registries_setting(&conn).await; + use windmill_common::global_settings as gs; + pass.option_setting( + JOB_DEFAULT_TIMEOUT_SECS_SETTING, + "JOB_DEFAULT_TIMEOUT_SECS", + JOB_DEFAULT_TIMEOUT.clone(), + ); + pass.setting(JOB_ISOLATION_SETTING, true, apply_job_isolation_setting); + pass.option_setting( + NSJAIL_TMPFS_SIZE_MB_SETTING, + "NSJAIL_TMPFS_SIZE_MB", + NSJAIL_TMPFS_SIZE_MB.clone(), + ); + pass.option_setting( + NSJAIL_TMP_BACKING_SETTING, + "NSJAIL_TMP_BACKING", + NSJAIL_TMP_BACKING.clone(), + ); + pass.option_setting( + SANDBOX_IMAGE_MAX_SIZE_MB_SETTING, + "SANDBOX_IMAGE_MAX_SIZE_MB", + SANDBOX_IMAGE_MAX_SIZE_MB.clone(), + ); + pass.option_setting( + SANDBOX_IMAGE_CACHE_MAX_MB_SETTING, + "SANDBOX_IMAGE_CACHE_MAX_MB", + SANDBOX_IMAGE_CACHE_MAX_MB.clone(), + ); + pass.option_setting( + SANDBOX_IMAGE_PULL_POLICY_SETTING, + "SANDBOX_IMAGE_PULL_POLICY", + SANDBOX_IMAGE_PULL_POLICY.clone(), + ); + pass.option_setting( + SANDBOX_IMAGE_DEFAULT_REGISTRY_SETTING, + "SANDBOX_IMAGE_DEFAULT_REGISTRY", + SANDBOX_IMAGE_DEFAULT_REGISTRY.clone(), + ); + pass.setting( + SANDBOX_REGISTRY_AUTH_SETTING, + true, + apply_sandbox_registry_auth_setting, + ); + pass.option_setting( + EXTRA_PIP_INDEX_URL_SETTING, + "PIP_EXTRA_INDEX_URL", + PIP_EXTRA_INDEX_URL.clone(), + ); + pass.option_setting( + PIP_INDEX_URL_SETTING, + "PIP_INDEX_URL", + PIP_INDEX_URL.clone(), + ); + pass.option_setting( + UV_INDEX_STRATEGY_SETTING, + "UV_INDEX_STRATEGY", + UV_INDEX_STRATEGY.clone(), + ); + pass.option_setting( + UV_EXCLUDE_NEWER_SETTING, + "UV_EXCLUDE_NEWER", + UV_EXCLUDE_NEWER.clone(), + ); + pass.option_setting( + UV_PYTHON_INSTALL_MIRROR_SETTING, + "UV_PYTHON_INSTALL_MIRROR", + UV_PYTHON_INSTALL_MIRROR.clone(), + ); + pass.option_setting( + BUN_INSTALL_MIN_RELEASE_AGE_SETTING, + "BUN_INSTALL_MIN_RELEASE_AGE", + BUN_INSTALL_MIN_RELEASE_AGE.clone(), + ); + pass.option_setting( + NPM_CONFIG_REGISTRY_SETTING, + "NPM_CONFIG_REGISTRY", + NPM_CONFIG_REGISTRY.clone(), + ); + pass.option_setting( + BUNFIG_INSTALL_SCOPES_SETTING, + "BUNFIG_INSTALL_SCOPES", + BUNFIG_INSTALL_SCOPES.clone(), + ); + pass.option_setting(NPMRC_SETTING, "NPMRC", NPMRC.clone()); + pass.option_setting( + INSTANCE_PYTHON_VERSION_SETTING, + "INSTANCE_PYTHON_VERSION", + INSTANCE_PYTHON_VERSION.clone(), + ); + pass.option_setting(NUGET_CONFIG_SETTING, "NUGET_CONFIG", NUGET_CONFIG.clone()); + pass.option_setting( + POWERSHELL_REPO_URL_SETTING, + "POWERSHELL_REPO_URL", + POWERSHELL_REPO_URL.clone(), + ); + pass.option_setting( + POWERSHELL_REPO_PAT_SETTING, + "POWERSHELL_REPO_PAT", + POWERSHELL_REPO_PAT.clone(), + ); + pass.option_setting(gs::MAVEN_REPOS_SETTING, "MAVEN_REPOS", MAVEN_REPOS.clone()); + pass.option_setting( + gs::MAVEN_SETTINGS_XML_SETTING, + "MAVEN_SETTINGS_XML", + MAVEN_SETTINGS_XML.clone(), + ); + pass.action(write_maven_settings_xml()); + pass.setting(gs::NO_DEFAULT_MAVEN_SETTING, true, |v| async move { + apply_no_default_maven_setting(v) + }); + pass.url_list_setting( + gs::RUBY_REPOS_SETTING, + "RUBY_REPOS", + windmill_worker::RUBY_REPOS.clone(), + ); + pass.option_setting( + gs::CARGO_REGISTRIES_SETTING, + "CARGO_REGISTRIES", + CARGO_REGISTRIES.clone(), + ); + pass.setting( + gs::WORKSPACE_REGISTRIES_SETTING, + true, + apply_workspace_registries_setting, + ); } + + pass.run(conn).await; } -pub async fn load_metrics_enabled(conn: &Connection) -> error::Result<()> { - let metrics_enabled = - load_value_from_global_settings_with_conn(conn, EXPOSE_METRICS_SETTING, true).await; - match metrics_enabled { - Ok(Some(serde_json::Value::Bool(t))) => METRICS_ENABLED.store(t, Ordering::Relaxed), - _ => (), - }; - Ok(()) + +pub fn apply_metrics_enabled(value: Option) { + if let Some(serde_json::Value::Bool(t)) = value { + METRICS_ENABLED.store(t, Ordering::Relaxed) + } } #[derive(serde::Deserialize)] @@ -564,23 +770,26 @@ pub async fn load_otel(db: &DB) { } pub async fn load_tag_per_workspace_enabled(db: &DB) -> error::Result<()> { - let metrics_enabled = - load_value_from_global_settings(db, DEFAULT_TAGS_PER_WORKSPACE_SETTING).await; - - match metrics_enabled { - Ok(Some(serde_json::Value::Bool(t))) => { - DEFAULT_TAGS_PER_WORKSPACE.store(t, Ordering::Relaxed) - } - _ => (), - }; + let v = load_value_from_global_settings(db, DEFAULT_TAGS_PER_WORKSPACE_SETTING).await?; + apply_tag_per_workspace_enabled(v); Ok(()) } -pub async fn load_tag_per_workspace_workspaces(db: &DB) -> error::Result<()> { - let workspaces = load_value_from_global_settings(db, DEFAULT_TAGS_WORKSPACES_SETTING).await; +pub fn apply_tag_per_workspace_enabled(value: Option) { + if let Some(serde_json::Value::Bool(t)) = value { + DEFAULT_TAGS_PER_WORKSPACE.store(t, Ordering::Relaxed) + } +} - match workspaces { - Ok(Some(serde_json::Value::Array(t))) => { +pub async fn load_tag_per_workspace_workspaces(db: &DB) -> error::Result<()> { + let v = load_value_from_global_settings(db, DEFAULT_TAGS_WORKSPACES_SETTING).await?; + apply_tag_per_workspace_workspaces(v); + Ok(()) +} + +pub fn apply_tag_per_workspace_workspaces(value: Option) { + match value { + Some(serde_json::Value::Array(t)) => { let workspaces = t .iter() .filter_map(|x| x.as_str()) @@ -588,24 +797,25 @@ pub async fn load_tag_per_workspace_workspaces(db: &DB) -> error::Result<()> { .collect::>(); DEFAULT_TAGS_WORKSPACES.store(std::sync::Arc::new(Some(workspaces))); } - Ok(None) => { + None => { DEFAULT_TAGS_WORKSPACES.store(std::sync::Arc::new(None)); } _ => (), }; - Ok(()) } pub async fn load_preview_tags_override(db: &DB) -> error::Result<()> { - let value = load_value_from_global_settings(db, PREVIEW_TAGS_OVERRIDE_SETTING).await; - - match value { - Ok(Some(serde_json::Value::Bool(t))) => PREVIEW_TAGS_OVERRIDE.store(t, Ordering::Relaxed), - _ => (), - }; + let v = load_value_from_global_settings(db, PREVIEW_TAGS_OVERRIDE_SETTING).await?; + apply_preview_tags_override(v); Ok(()) } +pub fn apply_preview_tags_override(value: Option) { + if let Some(serde_json::Value::Bool(t)) = value { + PREVIEW_TAGS_OVERRIDE.store(t, Ordering::Relaxed) + } +} + // Upper bound on the duration window. Postgres `make_interval(secs => $1::int4)` is the consumer // downstream, so this stays comfortably below `i32::MAX` and the subsequent `u32 -> i32` cast in // `workspace_fairness::refresh_overloaded` cannot wrap into a negative interval (which would @@ -635,12 +845,17 @@ pub async fn load_workspace_fairness_enabled(db: &DB) -> error::Result<()> { // atomic untouched rather than silently toggling the feature off across the whole cluster // (which would also trigger an unnecessary `store_pull_query` rebuild — exactly when DB load // is probably highest). - let new_enabled = - match load_value_from_global_settings(db, WORKSPACE_FAIRNESS_ENABLED_SETTING).await? { - Some(serde_json::Value::Bool(t)) => t, - // Setting unset / non-bool → explicit off. - _ => false, - }; + let v = load_value_from_global_settings(db, WORKSPACE_FAIRNESS_ENABLED_SETTING).await?; + apply_workspace_fairness_enabled(v).await; + Ok(()) +} + +pub async fn apply_workspace_fairness_enabled(value: Option) { + let new_enabled = match value { + Some(serde_json::Value::Bool(t)) => t, + // Setting unset / non-bool → explicit off. + _ => false, + }; let prev = WORKSPACE_FAIRNESS_ENABLED.swap(new_enabled, Ordering::Relaxed); // Re-store the pull queries so the fairness variants appear/disappear in // lockstep with the toggle. @@ -648,18 +863,23 @@ pub async fn load_workspace_fairness_enabled(db: &DB) -> error::Result<()> { let wc = windmill_common::worker::WORKER_CONFIG.load_full(); store_pull_query(&wc).await; } - Ok(()) } pub async fn load_workspace_fairness_max_percent(db: &DB) -> error::Result<()> { - // Distinguish three outcomes: - // - `Err(_)`: transient DB issue. Leave the atomic alone (don't clobber a known-good value - // because of a network blip during a notify-event propagation). - // - `Ok(None)` or `Ok(Some(invalid))`: setting is unset / explicitly cleared / corrupt. - // Restore the default so a deletion via the admin UI actually takes effect at runtime - // instead of leaving the stale in-memory value pinned until restart. - // - `Ok(Some(valid))`: clamp and store. - match load_value_from_global_settings(db, WORKSPACE_FAIRNESS_MAX_PERCENT_SETTING).await? { + let v = load_value_from_global_settings(db, WORKSPACE_FAIRNESS_MAX_PERCENT_SETTING).await?; + apply_workspace_fairness_max_percent(v); + Ok(()) +} + +// Distinguish three outcomes: +// - `Err(_)`: transient DB issue. Leave the atomic alone (don't clobber a known-good value +// because of a network blip during a notify-event propagation). +// - `Ok(None)` or `Ok(Some(invalid))`: setting is unset / explicitly cleared / corrupt. +// Restore the default so a deletion via the admin UI actually takes effect at runtime +// instead of leaving the stale in-memory value pinned until restart. +// - `Ok(Some(valid))`: clamp and store. +pub fn apply_workspace_fairness_max_percent(value: Option) { + match value { Some(serde_json::Value::Number(n)) => { let v = n .as_u64() @@ -672,12 +892,17 @@ pub async fn load_workspace_fairness_max_percent(db: &DB) -> error::Result<()> { .store(WORKSPACE_FAIRNESS_MAX_PERCENT_DEFAULT, Ordering::Relaxed); } } - Ok(()) } pub async fn load_workspace_fairness_duration_secs(db: &DB) -> error::Result<()> { - // See `load_workspace_fairness_max_percent` for the Err / None / invalid policy. - match load_value_from_global_settings(db, WORKSPACE_FAIRNESS_DURATION_SECS_SETTING).await? { + let v = load_value_from_global_settings(db, WORKSPACE_FAIRNESS_DURATION_SECS_SETTING).await?; + apply_workspace_fairness_duration_secs(v); + Ok(()) +} + +// See `load_workspace_fairness_max_percent` for the Err / None / invalid policy. +pub fn apply_workspace_fairness_duration_secs(value: Option) { + match value { Some(serde_json::Value::Number(n)) => { // Clamp to the safe range before narrowing. The downstream `u32 -> i32` cast in // `workspace_fairness::refresh_overloaded` makes any value above `i32::MAX` toxic @@ -693,12 +918,17 @@ pub async fn load_workspace_fairness_duration_secs(db: &DB) -> error::Result<()> .store(WORKSPACE_FAIRNESS_DURATION_SECS_DEFAULT, Ordering::Relaxed); } } - Ok(()) } pub async fn load_workspace_fairness_min_total(db: &DB) -> error::Result<()> { - // See `load_workspace_fairness_max_percent` for the Err / None / invalid policy. - match load_value_from_global_settings(db, WORKSPACE_FAIRNESS_MIN_TOTAL_SETTING).await? { + let v = load_value_from_global_settings(db, WORKSPACE_FAIRNESS_MIN_TOTAL_SETTING).await?; + apply_workspace_fairness_min_total(v); + Ok(()) +} + +// See `load_workspace_fairness_max_percent` for the Err / None / invalid policy. +pub fn apply_workspace_fairness_min_total(value: Option) { + match value { Some(serde_json::Value::Number(n)) => { // Clamp before narrowing — same reasoning as `_duration_secs`, just for the // counting threshold rather than the interval. @@ -713,12 +943,17 @@ pub async fn load_workspace_fairness_min_total(db: &DB) -> error::Result<()> { .store(WORKSPACE_FAIRNESS_MIN_TOTAL_DEFAULT, Ordering::Relaxed); } } - Ok(()) } pub async fn load_concurrency_key_max_queued(db: &DB) -> error::Result<()> { - // See `load_workspace_fairness_max_percent` for the Err / None / invalid policy. - match load_value_from_global_settings(db, CONCURRENCY_KEY_MAX_QUEUED_SETTING).await? { + let v = load_value_from_global_settings(db, CONCURRENCY_KEY_MAX_QUEUED_SETTING).await?; + apply_concurrency_key_max_queued(v); + Ok(()) +} + +// See `load_workspace_fairness_max_percent` for the Err / None / invalid policy. +pub fn apply_concurrency_key_max_queued(value: Option) { + match value { Some(serde_json::Value::Number(n)) => { // `0` is a meaningful value here (disable the cap), so unlike the fairness knobs // the lower bound is 0 rather than 1. @@ -746,7 +981,6 @@ pub async fn load_concurrency_key_max_queued(db: &DB) -> error::Result<()> { CONCURRENCY_KEY_MAX_QUEUED.store(CONCURRENCY_KEY_MAX_QUEUED_DEFAULT, Ordering::Relaxed); } } - Ok(()) } pub async fn load_workspace_max_queued_jobs(db: &DB) -> error::Result<()> { @@ -754,8 +988,14 @@ pub async fn load_workspace_max_queued_jobs(db: &DB) -> error::Result<()> { if !*CLOUD_HOSTED { return Ok(()); } - // Same Err / None / invalid policy as load_concurrency_key_max_queued: 0 disables. - match load_value_from_global_settings(db, WORKSPACE_MAX_QUEUED_JOBS_SETTING).await? { + let v = load_value_from_global_settings(db, WORKSPACE_MAX_QUEUED_JOBS_SETTING).await?; + apply_workspace_max_queued_jobs(v); + Ok(()) +} + +/// Same Err / None / invalid policy as [`apply_concurrency_key_max_queued`]: 0 disables. +pub fn apply_workspace_max_queued_jobs(value: Option) { + match value { Some(serde_json::Value::Number(n)) => { let v = n .as_u64() @@ -779,52 +1019,67 @@ pub async fn load_workspace_max_queued_jobs(db: &DB) -> error::Result<()> { WORKSPACE_MAX_QUEUED_JOBS.store(WORKSPACE_MAX_QUEUED_JOBS_DEFAULT, Ordering::Relaxed); } } - Ok(()) } pub async fn load_fork_workspace_tag_append_fork_suffix(db: &DB) -> error::Result<()> { - let value = - load_value_from_global_settings(db, FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX_SETTING).await; - - match value { - Ok(Some(serde_json::Value::Bool(t))) => { - FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX.store(t, Ordering::Relaxed) - } - Ok(None) => FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX.store(false, Ordering::Relaxed), - _ => (), - }; + let v = + load_value_from_global_settings(db, FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX_SETTING).await?; + apply_fork_workspace_tag_append_fork_suffix(v); Ok(()) } +pub fn apply_fork_workspace_tag_append_fork_suffix(value: Option) { + match value { + Some(serde_json::Value::Bool(t)) => { + FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX.store(t, Ordering::Relaxed) + } + None => FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX.store(false, Ordering::Relaxed), + _ => (), + }; +} + pub async fn reload_critical_alert_mute_ui_setting(conn: &Connection) -> error::Result<()> { - if let Ok(Some(serde_json::Value::Bool(t))) = - load_value_from_global_settings_with_conn(conn, CRITICAL_ALERT_MUTE_UI_SETTING, true).await - { + let v = + load_value_from_global_settings_with_conn(conn, CRITICAL_ALERT_MUTE_UI_SETTING, true).await?; + apply_critical_alert_mute_ui_setting(v); + Ok(()) +} + +pub fn apply_critical_alert_mute_ui_setting(value: Option) { + if let Some(serde_json::Value::Bool(t)) = value { CRITICAL_ALERT_MUTE_UI_ENABLED.store(t, Ordering::Relaxed); } - Ok(()) } pub async fn reload_critical_alerts_on_token_expiry_setting( conn: &Connection, ) -> error::Result<()> { - if let Ok(Some(serde_json::Value::Bool(t))) = load_value_from_global_settings_with_conn( + let v = load_value_from_global_settings_with_conn( conn, CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING, true, ) - .await - { - CRITICAL_ALERTS_ON_TOKEN_EXPIRY.store(t, Ordering::Relaxed); - } + .await?; + apply_critical_alerts_on_token_expiry_setting(v); Ok(()) } +pub fn apply_critical_alerts_on_token_expiry_setting(value: Option) { + if let Some(serde_json::Value::Bool(t)) = value { + CRITICAL_ALERTS_ON_TOKEN_EXPIRY.store(t, Ordering::Relaxed); + } +} + pub async fn load_metrics_debug_enabled(conn: &Connection) -> error::Result<()> { - let metrics_enabled = - load_value_from_global_settings_with_conn(conn, EXPOSE_DEBUG_METRICS_SETTING, true).await; - match metrics_enabled { - Ok(Some(serde_json::Value::Bool(t))) => { + let v = + load_value_from_global_settings_with_conn(conn, EXPOSE_DEBUG_METRICS_SETTING, true).await?; + apply_metrics_debug_enabled(v); + Ok(()) +} + +pub fn apply_metrics_debug_enabled(value: Option) { + match value { + Some(serde_json::Value::Bool(t)) => { METRICS_DEBUG_ENABLED.store(t, Ordering::Relaxed); //_RJEM_MALLOC_CONF=prof:true,prof_active:false,lg_prof_interval:30,lg_prof_sample:21,prof_prefix:/tmp/jeprof #[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))] @@ -836,7 +1091,6 @@ pub async fn load_metrics_debug_enabled(conn: &Connection) -> error::Result<()> } _ => (), }; - Ok(()) } #[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))] @@ -1132,16 +1386,18 @@ fn read_log_counters(ts_str: String) -> (usize, usize) { } pub async fn load_keep_job_dir(conn: &Connection) { - let value = load_value_from_global_settings_with_conn(conn, KEEP_JOB_DIR_SETTING, true).await; - match value { - Ok(Some(serde_json::Value::Bool(t))) => KEEP_JOB_DIR.store(t, Ordering::Relaxed), - Err(e) => { - tracing::error!("Error loading keep job dir metrics: {e:#}"); - } - _ => (), + match load_value_from_global_settings_with_conn(conn, KEEP_JOB_DIR_SETTING, true).await { + Ok(v) => apply_keep_job_dir(v), + Err(e) => tracing::error!("Error loading keep job dir metrics: {e:#}"), }; } +pub fn apply_keep_job_dir(value: Option) { + if let Some(serde_json::Value::Bool(t)) = value { + KEEP_JOB_DIR.store(t, Ordering::Relaxed) + } +} + pub async fn reload_otel_tracing_proxy_setting(conn: &Connection) { match load_value_from_global_settings_with_conn(conn, OTEL_TRACING_PROXY_SETTING, true).await { Ok(Some(settings)) => match serde_json::from_value::(settings) { @@ -1176,27 +1432,29 @@ pub async fn reload_otel_tracing_proxy_setting(conn: &Connection) { } pub async fn load_require_preexisting_user(db: &DB) { - let value = - load_value_from_global_settings(db, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING).await; - match value { - Ok(Some(serde_json::Value::Bool(t))) => { - REQUIRE_PREEXISTING_USER_FOR_OAUTH.store(t, Ordering::Relaxed) - } - Err(e) => { - tracing::error!("Error loading keep job dir metrics: {e:#}"); - } - _ => (), + match load_value_from_global_settings(db, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING).await { + Ok(v) => apply_require_preexisting_user(v), + Err(e) => tracing::error!("Error loading require_preexisting_user setting: {e:#}"), }; } +pub fn apply_require_preexisting_user(value: Option) { + if let Some(serde_json::Value::Bool(t)) = value { + REQUIRE_PREEXISTING_USER_FOR_OAUTH.store(t, Ordering::Relaxed) + } +} + pub async fn load_disable_password_login(db: &DB) { - let value = load_value_from_global_settings(db, DISABLE_PASSWORD_LOGIN_SETTING).await; + match load_value_from_global_settings(db, DISABLE_PASSWORD_LOGIN_SETTING).await { + Ok(v) => apply_disable_password_login(v), + Err(e) => tracing::error!("Error loading disable_password_login setting: {e:#}"), + }; +} + +pub fn apply_disable_password_login(value: Option) { match value { - Ok(Some(serde_json::Value::Bool(t))) => DISABLE_PASSWORD_LOGIN.store(t, Ordering::Relaxed), - Ok(None) => DISABLE_PASSWORD_LOGIN.store(false, Ordering::Relaxed), - Err(e) => { - tracing::error!("Error loading disable_password_login setting: {e:#}"); - } + Some(serde_json::Value::Bool(t)) => DISABLE_PASSWORD_LOGIN.store(t, Ordering::Relaxed), + None => DISABLE_PASSWORD_LOGIN.store(false, Ordering::Relaxed), _ => (), }; } @@ -2203,22 +2461,25 @@ async fn delete_log_files_from_disk_and_store( pub async fn reload_instance_events_webhook_setting(db: &DB) { use windmill_common::global_settings::INSTANCE_EVENTS_WEBHOOK_SETTING; - use windmill_common::webhook::INSTANCE_EVENTS_WEBHOOK; - let value = load_value_from_global_settings(db, INSTANCE_EVENTS_WEBHOOK_SETTING).await; + match load_value_from_global_settings(db, INSTANCE_EVENTS_WEBHOOK_SETTING).await { + Ok(v) => apply_instance_events_webhook_setting(v), + Err(e) => tracing::error!("Error loading instance_events_webhook setting: {e:#}"), + } +} + +pub fn apply_instance_events_webhook_setting(value: Option) { + use windmill_common::webhook::INSTANCE_EVENTS_WEBHOOK; match value { - Ok(Some(serde_json::Value::String(s))) if !s.is_empty() => { + Some(serde_json::Value::String(s)) if !s.is_empty() => { INSTANCE_EVENTS_WEBHOOK.store(std::sync::Arc::new(Some(s))); } - Ok(None) | Ok(Some(serde_json::Value::Null)) | Ok(Some(serde_json::Value::String(_))) => { + None | Some(serde_json::Value::Null) | Some(serde_json::Value::String(_)) => { // Fall back to env var if DB has no value INSTANCE_EVENTS_WEBHOOK.store(std::sync::Arc::new( std::env::var("INSTANCE_EVENTS_WEBHOOK").ok(), )); } - Err(e) => { - tracing::error!("Error loading instance_events_webhook setting: {e:#}"); - } _ => (), }; } @@ -2238,15 +2499,6 @@ pub async fn reload_timeout_wait_result_setting(conn: &Connection) { .await; } -pub async fn reload_saml_metadata_setting(conn: &Connection) { - reload_option_setting_with_tracing( - conn, - SAML_METADATA_SETTING, - "SAML_METADATA", - SAML_METADATA.clone(), - ) - .await; -} pub async fn reload_extra_pip_index_url_setting(conn: &Connection) { reload_option_setting_with_tracing( @@ -2338,9 +2590,6 @@ pub async fn reload_bunfig_install_scopes_setting(conn: &Connection) { .await; } -pub async fn reload_npmrc_setting(conn: &Connection) { - reload_option_setting_with_tracing(conn, NPMRC_SETTING, "NPMRC", NPMRC.clone()).await; -} pub async fn reload_nuget_config_setting(conn: &Connection) { reload_option_setting_with_tracing( @@ -2390,7 +2639,12 @@ pub async fn reload_maven_settings_xml_setting(conn: &Connection) { MAVEN_SETTINGS_XML.clone(), ) .await; + write_maven_settings_xml().await; +} +/// The half of [`reload_maven_settings_xml_setting`] after the read: mirrors the loaded value +/// onto disk, where the Maven CLI looks for it. +pub async fn write_maven_settings_xml() { if !cfg!(feature = "enterprise") { return; } @@ -2416,21 +2670,24 @@ pub async fn reload_maven_settings_xml_setting(conn: &Connection) { } pub async fn reload_no_default_maven_setting(conn: &Connection) { - let value = load_value_from_global_settings_with_conn( + match load_value_from_global_settings_with_conn( conn, windmill_common::global_settings::NO_DEFAULT_MAVEN_SETTING, true, ) - .await; - match value { - Ok(Some(serde_json::Value::Bool(t))) => NO_DEFAULT_MAVEN.store(t, Ordering::Relaxed), - Err(e) => { - tracing::error!("Error loading no default maven repository: {e:#}"); - } - _ => (), + .await + { + Ok(v) => apply_no_default_maven_setting(v), + Err(e) => tracing::error!("Error loading no default maven repository: {e:#}"), }; } +pub fn apply_no_default_maven_setting(value: Option) { + if let Some(serde_json::Value::Bool(t)) = value { + NO_DEFAULT_MAVEN.store(t, Ordering::Relaxed) + } +} + pub async fn reload_ruby_repos_setting(conn: &Connection) { reload_url_list_setting_with_tracing( conn, @@ -2441,25 +2698,23 @@ pub async fn reload_ruby_repos_setting(conn: &Connection) { .await; } -pub async fn reload_cargo_registries_setting(conn: &Connection) { - reload_option_setting_with_tracing( - conn, - windmill_common::global_settings::CARGO_REGISTRIES_SETTING, - "CARGO_REGISTRIES", - CARGO_REGISTRIES.clone(), - ) - .await; -} pub async fn reload_workspace_registries_setting(conn: &Connection) { - let value = load_value_from_global_settings_with_conn( + match load_value_from_global_settings_with_conn( conn, windmill_common::global_settings::WORKSPACE_REGISTRIES_SETTING, true, ) - .await; + .await + { + Ok(v) => apply_workspace_registries_setting(v).await, + Err(e) => tracing::error!("Error loading workspace_registries setting: {e:#}"), + } +} + +pub async fn apply_workspace_registries_setting(value: Option) { match value { - Ok(Some(v)) => match serde_json::from_value::(v) { + Some(v) => match serde_json::from_value::(v) { Ok(parsed) => { tracing::info!( "Loaded workspace registries for {} workspaces", @@ -2471,12 +2726,9 @@ pub async fn reload_workspace_registries_setting(conn: &Connection) { tracing::error!("Error parsing workspace_registries setting: {e:#}"); } }, - Ok(None) => { + None => { *WORKSPACE_REGISTRIES.write().await = None; } - Err(e) => { - tracing::error!("Error loading workspace_registries setting: {e:#}"); - } } } @@ -2622,16 +2874,14 @@ pub async fn reload_sandbox_registry_auth_setting(conn: &Connection) { // Secret-aware: the value is a raw docker/podman auth.json with credentials, so // it must never be logged. Load directly (the generic reload_option_setting path // logs the value via load_option_setting_value) and only log a redacted message. - let q = - match load_value_from_global_settings_with_conn(conn, SANDBOX_REGISTRY_AUTH_SETTING, true) - .await - { - Ok(q) => q, - Err(e) => { - tracing::error!("Error reloading setting SANDBOX_REGISTRY_AUTH: {e:?}"); - return; - } - }; + match load_value_from_global_settings_with_conn(conn, SANDBOX_REGISTRY_AUTH_SETTING, true).await + { + Ok(q) => apply_sandbox_registry_auth_setting(q).await, + Err(e) => tracing::error!("Error reloading setting SANDBOX_REGISTRY_AUTH: {e:?}"), + } +} + +pub async fn apply_sandbox_registry_auth_setting(q: Option) { let value = q.and_then(|q| serde_json::from_value::(q).ok()); let configured = value.as_ref().is_some_and(|v| !v.trim().is_empty()); *SANDBOX_REGISTRY_AUTH.write().await = value; @@ -2639,15 +2889,17 @@ pub async fn reload_sandbox_registry_auth_setting(conn: &Connection) { } pub async fn reload_job_isolation_setting(conn: &Connection) { - let value = - match load_value_from_global_settings_with_conn(conn, JOB_ISOLATION_SETTING, true).await { - Ok(Some(v)) => JobIsolationLevel::from_str(v.as_str().unwrap_or("")), - Ok(None) => JobIsolationLevel::Undefined, - Err(e) => { - tracing::error!("Error reloading job_isolation setting: {:?}", e); - return; - } - }; + match load_value_from_global_settings_with_conn(conn, JOB_ISOLATION_SETTING, true).await { + Ok(v) => apply_job_isolation_setting(v).await, + Err(e) => tracing::error!("Error reloading job_isolation setting: {:?}", e), + } +} + +pub async fn apply_job_isolation_setting(value: Option) { + let value = match value { + Some(v) => JobIsolationLevel::from_str(v.as_str().unwrap_or("")), + None => JobIsolationLevel::Undefined, + }; let old_value = JobIsolationLevel::from_u8(JOB_ISOLATION.swap(value as u8, Ordering::Relaxed)); if old_value != value { tracing::info!( @@ -2670,20 +2922,6 @@ pub async fn reload_job_isolation_setting(conn: &Connection) { } } -pub async fn reload_request_size(conn: &Connection) { - if let Err(e) = reload_setting( - conn, - REQUEST_SIZE_LIMIT_SETTING, - "REQUEST_SIZE_LIMIT", - DEFAULT_BODY_LIMIT, - REQUEST_SIZE_LIMIT.clone(), - |x| x.mul(1024 * 1024), - ) - .await - { - tracing::error!("Error reloading retention period: {:?}", e) - } -} async fn resolve_license_key_value(conn: &Connection, quiet: bool) -> anyhow::Result { let q = load_value_from_global_settings_with_conn(conn, LICENSE_KEY_SETTING, true) @@ -2792,18 +3030,306 @@ pub async fn reload_option_setting_with_tracing( } } -pub async fn load_value_from_global_settings( - db: &DB, - setting_name: &str, -) -> error::Result> { - let r = sqlx::query!( - "SELECT value FROM global_settings WHERE name = $1", - setting_name - ) - .fetch_optional(db) - .await? - .map(|x| x.value); - Ok(r) +type SettingApplier<'a> = + Box) -> BoxFuture<'a, ()> + Send + 'a>; + +/// The values a [`PassStep::Settings`] step asked for, keyed by setting name. +pub type SettingValues = std::collections::HashMap<&'static str, Option>; + +type MultiSettingApplier<'a> = Box BoxFuture<'a, ()> + Send + 'a>; + +enum PassStep<'a> { + /// A setting to read, paired with what to do once its value is in hand. + Setting { name: &'static str, http: bool, apply: SettingApplier<'a> }, + /// Several settings one piece of work needs together. + Settings { names: &'static [&'static str], http: bool, apply: MultiSettingApplier<'a> }, + /// Work that is not a settings read but has to keep its place in the sequence. + Action(BoxFuture<'a, ()>), +} + +/// One settings-loading pass: every read it will make, declared up front, then fetched +/// together and applied in the order they were declared. +/// +/// A pass that reads several dozen settings costs one round trip instead of one each, which +/// is invisible against a local database and is the bulk of a worker's startup latency +/// against a real one. Declaring is what makes the batch exact: the same `if server_mode` / +/// `if *CLOUD_HOSTED` branches that used to guard a read now guard a [`Self::setting`] call, +/// so the fetch asks for what this process actually needs and nothing else. +/// +/// Order is preserved end to end, which is what lets non-setting work sit in the middle of the +/// sequence via [`Self::action`]: appliers run in declaration order, so a setting whose applier +/// depends on an earlier one having landed still sees it. +pub struct SettingsPass<'a> { + steps: Vec>, +} + +impl<'a> SettingsPass<'a> { + pub fn new() -> Self { + SettingsPass { steps: Vec::new() } + } + + /// Declare a read. `http` mirrors `load_from_http` in + /// [`load_value_from_global_settings_with_conn`]: `false` means an agent worker leaves the + /// setting unset rather than asking the server for it. + pub fn setting(&mut self, name: &'static str, http: bool, apply: F) + where + F: FnOnce(Option) -> Fut + Send + 'a, + Fut: std::future::Future + Send + 'a, + { + self.steps.push(PassStep::Setting { + name, + http, + apply: Box::new(move |v| Box::pin(apply(v))), + }); + } + + /// Declare a step that needs several settings at once. It is skipped, like a single + /// [`Self::setting`], if any of them could not be read. + pub fn settings(&mut self, names: &'static [&'static str], http: bool, apply: F) + where + F: FnOnce(SettingValues) -> Fut + Send + 'a, + Fut: std::future::Future + Send + 'a, + { + self.steps.push(PassStep::Settings { + names, + http, + apply: Box::new(move |v| Box::pin(apply(v))), + }); + } + + /// Declare work that is not a settings read, keeping its position in the sequence. + pub fn action(&mut self, fut: Fut) + where + Fut: std::future::Future + Send + 'a, + { + self.steps.push(PassStep::Action(Box::pin(fut))); + } + + /// The batched form of [`reload_option_setting_with_tracing`]. + pub fn option_setting( + &mut self, + name: &'static str, + std_env_var: &'static str, + lock: Arc>>, + ) { + self.option_setting_with(name, std_env_var, move |v| async move { + *lock.write().await = v; + }); + } + + /// [`Self::option_setting`] for a setting held in something other than an + /// `Arc>>`, such as an `ArcSwap`. Going through here rather than calling + /// [`parse_option_setting_value`] from a bare [`Self::setting`] is what applies the + /// `FORCE_` rule, which a hand-rolled declaration would silently miss. + pub fn option_setting_with( + &mut self, + name: &'static str, + std_env_var: &'static str, + store: F, + ) where + T: FromStr + DeserializeOwned + Send + Sync + 'a, + F: FnOnce(Option) -> Fut + Send + 'a, + Fut: std::future::Future + Send + 'a, + { + if forced_env_value::(std_env_var).is_some() { + self.forced(move || async move { + store(parse_option_setting_value(None, name, std_env_var)).await + }); + return; + } + self.setting(name, true, move |v| async move { + store(parse_option_setting_value(v, name, std_env_var)).await + }); + } + + /// A setting with a default, the batched counterpart of [`load_setting_value`]. + pub fn required_setting( + &mut self, + name: &'static str, + std_env_var: &'static str, + default: T, + lock: Arc>, + transformer: fn(T) -> T, + ) { + self.setting(name, true, move |v| async move { + *lock.write().await = parse_setting_value(v, name, std_env_var, default, transformer); + }); + } + + /// The batched form of [`reload_url_list_setting_with_tracing`]. + pub fn url_list_setting( + &mut self, + name: &'static str, + std_env_var: &'static str, + lock: Arc>>>, + ) { + if std::env::var(format!("FORCE_{}", std_env_var)).is_ok() { + self.forced(move || async move { + *lock.write().await = parse_url_list_setting_value(None, name, std_env_var); + }); + return; + } + self.setting(name, true, move |v| async move { + *lock.write().await = parse_url_list_setting_value(v, name, std_env_var); + }); + } + + /// Apply a setting whose value comes from a `FORCE_` override. + /// + /// Declared as a step with no read: the override outranks the database, so fetching is + /// pointless, and more importantly a failed fetch must not drop it. A skipped applier is + /// how an unreadable setting keeps its current value, which for a forced one would mean + /// silently running unforced. + fn forced(&mut self, apply: F) + where + F: FnOnce() -> Fut + Send + 'a, + Fut: std::future::Future + Send + 'a, + { + self.action(async move { apply().await }); + } + + /// Fetch every declared setting in one go, then run the steps in declaration order. + pub async fn run(self, conn: &Connection) { + let over_http = matches!(conn, Connection::Http(_)); + let declared: Vec<(&'static str, bool)> = self + .steps + .iter() + .flat_map(|s| match s { + PassStep::Setting { name, http, .. } => vec![(*name, *http)], + PassStep::Settings { names, http, .. } => { + names.iter().map(|n| (*n, *http)).collect() + } + PassStep::Action(_) => vec![], + }) + .collect(); + + // A setting an agent worker is not allowed to ask the server for reads as unset, which + // is what `load_value_from_global_settings_with_conn(.., false)` returned for it. That + // is not the same as a read that failed, which is left out and skips its applier. + let names: Vec<&str> = declared + .iter() + .filter_map(|(name, http)| (!over_http || *http).then_some(*name)) + .collect(); + + let mut values = fetch_settings_batch(conn, &names).await; + // One failed query took every setting with it. At startup there is no known-good + // in-memory state to preserve, so read them individually rather than leave the process + // on compile-time defaults until the next full reload. Only the single-query transport + // can fail this way; over HTTP the batch already is the per-setting read. + if matches!(conn, Connection::Sql(_)) && values.is_empty() && !names.is_empty() { + tracing::warn!("Falling back to per-setting reads for {} settings", names.len()); + values = fetch_settings_individually(conn, &names).await; + } + for (name, http) in &declared { + if over_http && !*http { + values.insert(name.to_string(), None); + } + } + + for step in self.steps { + match step { + // A name missing from `values` is one whose read failed, not one that is + // unset. Skipping its applier is what keeps a transient database error from + // looking like a cleared setting: several of them reset to a default on + // `None`, and would otherwise clobber a known-good value on a blip. + PassStep::Setting { name, apply, .. } => { + if let Some(value) = values.remove(name) { + apply(value).await + } + } + PassStep::Settings { names, apply, .. } => { + let asked: SettingValues = names + .iter() + .filter_map(|n| values.get(*n).map(|v| (*n, v.clone()))) + .collect(); + if asked.len() == names.len() { + apply(asked).await + } + } + PassStep::Action(fut) => fut.await, + } + } + } +} + +/// The value of a `FORCE_` override, when it is set and parses. +/// +/// Mirrors the check [`load_option_setting_value`] makes before it reads, so the batched and +/// per-setting paths agree on when the database is consulted at all. +fn forced_env_value(std_env_var: &str) -> Option { + std::env::var(format!("FORCE_{}", std_env_var)) + .ok() + .and_then(|x| x.parse::().ok()) +} + +/// Parse whitespace-separated URLs, dropping and reporting the ones that do not parse. +fn parse_url_list(raw: &str, source: &str) -> Vec { + raw.trim() + .split_whitespace() + .filter_map(|url_str| match url::Url::parse(url_str) { + Ok(url) => Some(url), + Err(e) => { + tracing::error!("Invalid URL in {}: '{}': {}", source, url_str, e); + None + } + }) + .collect() +} + +/// One read per setting, concurrently. The fallback for a batch that failed as a whole: a +/// name whose own read also fails is left out, so its applier is skipped rather than told the +/// setting is unset. +async fn fetch_settings_individually( + conn: &Connection, + names: &[&str], +) -> std::collections::HashMap> { + futures::future::join_all(names.iter().map(|name| async move { + ( + name.to_string(), + load_value_from_global_settings_with_conn(conn, name, true).await, + ) + })) + .await + .into_iter() + .filter_map(|(name, value)| match value { + Ok(value) => Some((name, value)), + Err(e) => { + tracing::error!("Error loading setting {name}: {e:#}"); + None + } + }) + .collect() +} + +/// Read many settings at once: one query over a database connection, and for an agent worker +/// one concurrent round of the per-setting endpoint rather than a sequential walk of it. +/// +/// A name whose read failed is left out entirely, which the caller distinguishes from a name +/// that is present with no value, i.e. genuinely unset. +async fn fetch_settings_batch( + conn: &Connection, + names: &[&str], +) -> std::collections::HashMap> { + match conn { + Connection::Sql(db) => { + match windmill_common::global_settings::load_values_from_global_settings(db, names) + .await + { + // Every requested name is accounted for: the ones with no row read as unset. + Ok(mut rows) => names + .iter() + .map(|name| (name.to_string(), rows.remove(*name))) + .collect(), + Err(e) => { + tracing::error!("Could not load global settings: {e:#}"); + std::collections::HashMap::new() + } + } + } + // An agent worker has no batch endpoint, but issuing the reads together still costs + // one round instead of one per setting. + Connection::Http(_) => fetch_settings_individually(conn, names).await, + } } pub async fn load_value_from_global_settings_with_conn( @@ -2848,6 +3374,22 @@ pub async fn load_option_setting_value( } let q = load_value_from_global_settings_with_conn(conn, setting_name, true).await?; + Ok(parse_option_setting_value(q, setting_name, std_env_var)) +} + +/// The half of [`load_option_setting_value`] after the read, so [`SettingsPass`] can parse a +/// value it already fetched. +pub fn parse_option_setting_value( + q: Option, + setting_name: &str, + std_env_var: &str, +) -> Option { + if let Some(force_value) = std::env::var(format!("FORCE_{}", std_env_var)) + .ok() + .and_then(|x| x.parse::().ok()) + { + return Some(force_value); + } let mut value = std::env::var(std_env_var) .ok() @@ -2866,7 +3408,7 @@ pub async fn load_option_setting_value( tracing::info!("Loaded {setting_name} setting to None"); } - Ok(value) + value } pub async fn reload_option_setting( @@ -2922,6 +3464,23 @@ pub async fn load_url_list_setting_value( } let q = load_value_from_global_settings_with_conn(conn, setting_name, true).await?; + Ok(parse_url_list_setting_value(q, setting_name, std_env_var)) +} + +/// The half of [`load_url_list_setting_value`] after the read, so [`SettingsPass`] can parse a +/// value it already fetched. +pub fn parse_url_list_setting_value( + q: Option, + setting_name: &str, + std_env_var: &str, +) -> Option> { + // A FORCE_ override wins over both the database and the ordinary env var. Invalid URLs in + // it are dropped with an error here rather than failing the read, since a settings pass + // has nowhere to return the failure to. + if let Ok(force_value) = std::env::var(format!("FORCE_{}", std_env_var)) { + let urls = parse_url_list(&force_value, &format!("FORCE_{}", std_env_var)); + return if urls.is_empty() { None } else { Some(urls) }; + } // Check regular environment variable let mut value = if let Ok(env_value) = std::env::var(std_env_var) { @@ -2972,7 +3531,7 @@ pub async fn load_url_list_setting_value( tracing::info!("Loaded {} setting to None", setting_name); } - Ok(value) + value } pub async fn reload_url_list_setting( @@ -2989,11 +3548,9 @@ pub async fn reload_url_list_setting( Ok(()) } -/// Load a required setting value without writing it anywhere. -/// -/// Extracted from [`reload_setting`] so callers that store the value in -/// something other than `Arc>` (e.g. `AtomicI64`, `AtomicBool`, -/// `ArcSwap`) can reuse the load pipeline. +/// Load a required setting value without writing it anywhere, so callers that store it in +/// something other than `Arc>` (e.g. `AtomicI64`, `AtomicBool`, `ArcSwap`) can +/// reuse the load pipeline. pub async fn load_setting_value( conn: &Connection, setting_name: &str, @@ -3002,7 +3559,24 @@ pub async fn load_setting_value( transformer: fn(T) -> T, ) -> error::Result { let q = load_value_from_global_settings_with_conn(conn, setting_name, true).await?; + Ok(parse_setting_value( + q, + setting_name, + std_env_var, + default, + transformer, + )) +} +/// The half of [`load_setting_value`] after the read, so [`SettingsPass`] can parse a value it +/// already fetched. +pub fn parse_setting_value( + q: Option, + setting_name: &str, + std_env_var: &str, + default: T, + transformer: fn(T) -> T, +) -> T { let mut value = std::env::var(std_env_var) .ok() .and_then(|x| x.parse::().ok()) @@ -3017,24 +3591,9 @@ pub async fn load_setting_value( } }; - Ok(value) + value } -pub async fn reload_setting( - conn: &Connection, - setting_name: &str, - std_env_var: &str, - default: T, - lock: Arc>, - transformer: fn(T) -> T, -) -> error::Result<()> { - let value = load_setting_value(conn, setting_name, std_env_var, default, transformer).await?; - { - let mut l = lock.write().await; - *l = value; - } - Ok(()) -} #[cfg(feature = "prometheus")] pub async fn monitor_pool(db: &DB) { @@ -3242,6 +3801,19 @@ pub async fn monitor_db( } }; + // Not gated on server_mode: feature-usage counters accumulate wherever an + // instrumented call site runs, and a worker that never flushed would lose + // its counts on shutdown. + let feature_usage_f = async { + if !initial_load { + if let Some(db) = conn.as_sql() { + if let Err(e) = windmill_common::feature_usage::flush_feature_usage(db).await { + tracing::error!("Error flushing feature_usage counters: {e}"); + } + } + } + }; + let verify_license_key_f = async { #[cfg(feature = "enterprise")] if !initial_load { @@ -3491,6 +4063,7 @@ pub async fn monitor_db( join!( expired_items_f, + feature_usage_f, zombie_jobs_f, stale_jobs_f, trim_resource_versions_f, @@ -4328,7 +4901,12 @@ pub async fn reload_worker_config(db: &DB, tx: KillpillSender, kill_if_change: b pub async fn load_base_url(conn: &Connection) -> error::Result { let q_base_url = load_value_from_global_settings_with_conn(conn, BASE_URL_SETTING, false).await?; + Ok(parse_base_url(q_base_url)) +} +/// The half of [`load_base_url`] after the read, so [`SettingsPass`] can use a value it +/// already fetched. Stores into `BASE_URL` as well as returning it. +pub fn parse_base_url(q_base_url: Option) -> String { let std_base_url = std::env::var("BASE_URL") .ok() .unwrap_or_else(|| "http://localhost".to_string()); @@ -4350,14 +4928,32 @@ pub async fn load_base_url(conn: &Connection) -> error::Result { std_base_url }; BASE_URL.store(std::sync::Arc::new(base_url.clone())); - Ok(base_url) + base_url } pub async fn reload_base_url_setting(conn: &Connection) -> error::Result<()> { #[cfg(feature = "oauth2")] - let oauths = if let Some(db) = conn.as_sql() { - let q_oauth = load_value_from_global_settings(db, OAUTH_SETTING).await?; + let q_oauth = match conn.as_sql() { + Some(db) => load_value_from_global_settings(db, OAUTH_SETTING).await?, + None => None, + }; + #[cfg(not(feature = "oauth2"))] + let q_oauth = None; + let q_base_url = + load_value_from_global_settings_with_conn(conn, BASE_URL_SETTING, false).await?; + apply_base_url_setting(conn, q_oauth, q_base_url).await +} +/// The half of [`reload_base_url_setting`] after the reads. +pub async fn apply_base_url_setting( + conn: &Connection, + q_oauth: Option, + q_base_url: Option, +) -> error::Result<()> { + // Both only reach a use under a feature gate. + let (_, _) = (&conn, &q_oauth); + #[cfg(feature = "oauth2")] + let oauths = if conn.as_sql().is_some() { if let Some(q) = q_oauth { if let Ok(v) = serde_json::from_value::< Option>, @@ -4374,7 +4970,7 @@ pub async fn reload_base_url_setting(conn: &Connection) -> error::Result<()> { } else { None }; - let base_url = load_base_url(conn).await?; + let base_url = parse_base_url(q_base_url); let is_secure = base_url.starts_with("https://"); #[cfg(feature = "oauth2")] @@ -5540,9 +6136,18 @@ pub async fn reload_hub_base_url_setting( conn: &Connection, server_mode: bool, ) -> error::Result<()> { - let hub_base_url = - load_value_from_global_settings_with_conn(conn, HUB_BASE_URL_SETTING, true).await?; + let v = load_value_from_global_settings_with_conn(conn, HUB_BASE_URL_SETTING, true).await?; + apply_hub_base_url_setting(conn, server_mode, v).await +} +/// The half of [`reload_hub_base_url_setting`] after the read. +pub async fn apply_hub_base_url_setting( + conn: &Connection, + server_mode: bool, + hub_base_url: Option, +) -> error::Result<()> { + // Only reaches a use under the `embedding` feature. + let _ = &conn; let base_url = if let Some(q) = hub_base_url { if let Ok(v) = serde_json::from_value::(q.clone()) { if v != "" { @@ -5586,9 +6191,12 @@ pub async fn reload_hub_base_url_setting( } pub async fn reload_critical_error_channels_setting(conn: &DB) -> error::Result<()> { - let critical_error_channels = - load_value_from_global_settings(conn, CRITICAL_ERROR_CHANNELS_SETTING).await?; + let v = load_value_from_global_settings(conn, CRITICAL_ERROR_CHANNELS_SETTING).await?; + apply_critical_error_channels_setting(v); + Ok(()) +} +pub fn apply_critical_error_channels_setting(critical_error_channels: Option) { let critical_error_channels = if let Some(q) = critical_error_channels { if let Ok(v) = serde_json::from_value::>(q.clone()) { v @@ -5604,14 +6212,15 @@ pub async fn reload_critical_error_channels_setting(conn: &DB) -> error::Result< }; CRITICAL_ERROR_CHANNELS.store(std::sync::Arc::new(critical_error_channels)); - - Ok(()) } pub async fn reload_app_workspaced_route_setting(conn: &DB) -> error::Result<()> { - let app_workspaced_route = - load_value_from_global_settings(conn, APP_WORKSPACED_ROUTE_SETTING).await?; + let v = load_value_from_global_settings(conn, APP_WORKSPACED_ROUTE_SETTING).await?; + apply_app_workspaced_route_setting(v); + Ok(()) +} +pub fn apply_app_workspaced_route_setting(app_workspaced_route: Option) { let ws_route = match app_workspaced_route { Some(serde_json::Value::Bool(ws_route)) => ws_route, None => false, @@ -5626,13 +6235,17 @@ pub async fn reload_app_workspaced_route_setting(conn: &DB) -> error::Result<()> }; APP_WORKSPACED_ROUTE.store(ws_route, Ordering::Relaxed); - Ok(()) } pub async fn reload_http_route_workspaced_route_setting(conn: &DB) -> error::Result<()> { - let http_route_workspaced_route = - load_value_from_global_settings(conn, HTTP_ROUTE_WORKSPACED_ROUTE_SETTING).await?; + let v = load_value_from_global_settings(conn, HTTP_ROUTE_WORKSPACED_ROUTE_SETTING).await?; + apply_http_route_workspaced_route_setting(conn, v).await +} +pub async fn apply_http_route_workspaced_route_setting( + conn: &DB, + http_route_workspaced_route: Option, +) -> error::Result<()> { let ws_route = match http_route_workspaced_route { Some(serde_json::Value::Bool(ws_route)) => ws_route, None => false, @@ -5689,30 +6302,34 @@ pub async fn reload_critical_alerts_on_db_oversize(conn: &DB) -> error::Result<( Ok(()) } -async fn generate_and_save_jwt_secret(db: &DB) -> error::Result { - let secret = rd_string(32); - sqlx::query!( - "INSERT INTO global_settings (name, value) VALUES ($1, $2) ON CONFLICT (name) DO UPDATE SET value = EXCLUDED.value", - JWT_SECRET_SETTING, - serde_json::to_value(&secret).unwrap() - ).execute(db).await?; - - Ok(secret) -} pub async fn reload_jwt_secret_setting(db: &DB) -> error::Result<()> { - let jwt_secret = load_value_from_global_settings(db, JWT_SECRET_SETTING).await?; + let v = load_value_from_global_settings(db, JWT_SECRET_SETTING).await?; + apply_jwt_secret_setting(db, v).await +} - let jwt_secret = if let Some(q) = jwt_secret { - if let Ok(v) = serde_json::from_value::(q.clone()) { - v - } else { - tracing::error!("Could not parse jwt_secret setting, generating new one"); - generate_and_save_jwt_secret(db).await? +/// The half of [`reload_jwt_secret_setting`] after the read. +/// +/// `value` may be stale, which is why generating falls to +/// [`get_or_create_jwt_secret`]: that statement, not this read, decides whether a new secret +/// is stored, so a pass that batched an absent read cannot overwrite one another process +/// wrote in the meantime. +pub async fn apply_jwt_secret_setting( + db: &DB, + value: Option, +) -> error::Result<()> { + let jwt_secret = match value { + Some(q) => match serde_json::from_value::(q) { + Ok(v) => v, + Err(_) => { + tracing::error!("Could not parse jwt_secret setting, generating new one"); + get_or_create_jwt_secret(db).await? + } + }, + None => { + tracing::info!("No jwt secret found, generating one"); + get_or_create_jwt_secret(db).await? } - } else { - tracing::info!("Not jwt secret found, generating one"); - generate_and_save_jwt_secret(db).await? }; JWT_SECRET.store(std::sync::Arc::new(jwt_secret)); diff --git a/backend/summarized_schema.txt b/backend/summarized_schema.txt index 44b62ca51e..732f1ece52 100644 --- a/backend/summarized_schema.txt +++ b/backend/summarized_schema.txt @@ -176,6 +176,9 @@ token: token_hash(char), token_prefix(char), token(char), label(char), expiratio FK: (workspace_id) -> workspace(id) token_expiry_notification: token_hash(char), expiration(ts) INDEX: idx_token_expiry_notification_expiration (expiration) +trigger_history: id(bigint), workspace_id(char), trigger_kind(char), path(char), operation(char), source(char), username(char), created_at(ts), changes(jsonb) + FK: (workspace_id) -> workspace(id) + INDEX: idx_trigger_history_workspace_kind_path (workspace_id, trigger_kind, path, id), idx_trigger_history_workspace_id (workspace_id, id) tutorial_progress: email(char), progress(bit64), skipped_all(bool) unique_ext_jwt_token: jwt_hash(bigint), last_used_at(ts), email(text), username(text), is_admin(bool), is_operator(bool), workspace_id(text?), label(text?), scopes(text[]?) usage: id(char), is_workspace(bool), month_(int), usage(int) diff --git a/backend/tests/fixtures/jobs_read_auth.sql b/backend/tests/fixtures/jobs_read_auth.sql index 03ee69b1f4..d7d11228aa 100644 --- a/backend/tests/fixtures/jobs_read_auth.sql +++ b/backend/tests/fixtures/jobs_read_auth.sql @@ -19,6 +19,76 @@ INSERT INTO token(token_hash, token_prefix, token, email, label, super_admin, sc ARRAY['jobs:read', 'if_jobs:filter_tags:deno'] ); +-- A path-scoped run token for test-user-2, as the trigger UI mints per runnable for a +-- webhook caller. test-user-2 created every job this token is asserted against, so the +-- `created_by` grant would otherwise hand it all of them; it must reach only jobs of +-- `f/shared/flow1`. +INSERT INTO token(token_hash, token_prefix, token, email, label, super_admin, scopes) VALUES ( + encode(sha256('RUN_SCOPED_TOKEN'::bytea), 'hex'), 'RUN_SCOPE', 'RUN_SCOPED_TOKEN', + 'test2@windmill.dev', 'flow webhook token', false, + ARRAY['jobs:run:flows:f/shared/flow1'] +); + +-- Same, scoped to a script. The two jobs below both run through a `singlestepflow` +-- wrapper (native retry / scheduled runs produce these) — one wrapping a script, one +-- wrapping a flow — so the confinement has to project each onto the runnable it wraps +-- rather than onto the wrapper's own `kind`. +INSERT INTO token(token_hash, token_prefix, token, email, label, super_admin, scopes) VALUES ( + encode(sha256('RUN_SCOPED_SCRIPT_TOKEN'::bytea), 'hex'), 'RUN_SCRIP', 'RUN_SCOPED_SCRIPT_TOKEN', + 'test2@windmill.dev', 'script webhook token', false, + ARRAY['jobs:run:scripts:u/test-user-2/wrapped_script'] +); + +INSERT INTO public.v2_job ( + id, workspace_id, created_by, created_at, permissioned_as, permissioned_as_email, + kind, script_lang, runnable_path, tag, visible_to_owner, raw_flow +) VALUES ( + '14141414-1414-1414-1414-141414141414', 'test-workspace', 'test-user-2', + '2023-01-01 00:00:00', 'u/test-user-2', 'test2@windmill.dev', + 'singlestepflow', 'deno', 'u/test-user-2/wrapped_script', 'deno', true, + '{"modules": [{"id": "a", "value": {"type": "script", "path": "u/test-user-2/wrapped_script"}}]}' +); +INSERT INTO public.v2_job_completed (id, workspace_id, duration_ms, status, result) VALUES + ('14141414-1414-1414-1414-141414141414', 'test-workspace', 1000, 'success'::job_status, + '{"wrapped": "WRAPPED_RESULT"}'); + +INSERT INTO public.v2_job ( + id, workspace_id, created_by, created_at, permissioned_as, permissioned_as_email, + kind, script_lang, runnable_path, tag, visible_to_owner, raw_flow +) VALUES ( + '15151515-1515-1515-1515-151515151515', 'test-workspace', 'test-user-2', + '2023-01-01 00:00:00', 'u/test-user-2', 'test2@windmill.dev', + 'singlestepflow', 'deno', 'f/shared/flow1', 'flow', true, + '{"modules": [{"id": "a", "value": {"type": "flow", "path": "f/shared/flow1"}}]}' +); +INSERT INTO public.v2_job_completed (id, workspace_id, duration_ms, status, result) VALUES + ('15151515-1515-1515-1515-151515151515', 'test-workspace', 1000, 'success'::job_status, + '{"wrapped": "WRAPPED_FLOW_RESULT"}'); + +-- A token pairing an app scope with a run scope, as someone driving an app's components +-- programmatically would build. `APP_INLINE_JOB` is an inline-script component run: no +-- `jobs:run` scope can name its kind, so only the `apps:run` half puts it in reach. +INSERT INTO token(token_hash, token_prefix, token, email, label, super_admin, scopes) VALUES ( + encode(sha256('APP_RUNNER_TOKEN'::bytea), 'hex'), 'APP_RUNNE', 'APP_RUNNER_TOKEN', + 'test2@windmill.dev', 'app runner token', false, + ARRAY['apps:run:u/test-user-2/dash', 'jobs:run:scripts:u/test-user-2/wrapped_script'] +); + +-- An inline-script component run of app `u/test-user-2/dash`, stamped with the +-- app provenance `execute_component` sets (`trigger_kind = 'app'`). +INSERT INTO public.v2_job ( + id, workspace_id, created_by, created_at, permissioned_as, permissioned_as_email, + kind, script_lang, runnable_path, tag, visible_to_owner, trigger_kind, trigger, args +) VALUES ( + '16161616-1616-1616-1616-161616161616', 'test-workspace', 'test-user-2', + '2023-01-01 00:00:00', 'u/test-user-2', 'test2@windmill.dev', + 'appscript', 'deno', NULL, 'deno', false, 'app', 'u/test-user-2/dash', + '{"component": "arg"}' +); +INSERT INTO public.v2_job_completed (id, workspace_id, duration_ms, status, result) VALUES + ('16161616-1616-1616-1616-161616161616', 'test-workspace', 1000, 'success'::job_status, + '{"inline": "APP_INLINE_RESULT"}'); + -- App embed token for the admin viewer (test-user). Mirrors a minted sandboxed -- low-code app token: carries the `app_embed` sentinel plus the embed scope set. -- Used to assert the token is confined to jobs the viewer LAUNCHED, not every job diff --git a/backend/tests/jobs_read_auth.rs b/backend/tests/jobs_read_auth.rs index 317b1c6866..c8c154c250 100644 --- a/backend/tests/jobs_read_auth.rs +++ b/backend/tests/jobs_read_auth.rs @@ -46,6 +46,12 @@ const RUNNING_JOB: &str = "77777777-7777-7777-7777-777777777777"; const EMBED_OWN_JOB: &str = "12121212-1212-1212-1212-121212121212"; // A QUEUED job launched by the embed viewer (created_by test-user) — cancelable by it. const EMBED_OWN_QUEUED: &str = "13131313-1313-1313-1313-131313131313"; +// `singlestepflow` wrappers (as native retry / scheduled runs produce), one around a +// SCRIPT and one around a FLOW. +const WRAPPED_JOB: &str = "14141414-1414-1414-1414-141414141414"; +const WRAPPED_FLOW_JOB: &str = "15151515-1515-1515-1515-151515151515"; +// An inline-script component run of app `u/test-user-2/dash` (`trigger_kind = 'app'`). +const APP_INLINE_JOB: &str = "16161616-1616-1616-1616-161616161616"; // Queued sub-flow test-user-3 can see (folder `shared`), whose parent top flow they // cannot. Force cancel walks up to that parent. const QUEUED_VISIBLE_MID: &str = "55555555-5555-5555-5555-555555555555"; @@ -380,6 +386,160 @@ async fn test_single_job_read_authorization(db: Pool) -> anyhow::Resul } } + // ---- PATH-SCOPED RUN TOKEN: confined to jobs of the runnable it may start. + // RUN_SCOPED_TOKEN is test-user-2's `jobs:run:flows:f/shared/flow1` webhook + // token, and test-user-2 created every job asserted on below — so `created_by` + // alone would hand it all of them. + // Its own flow run reads, and so do the steps beneath it: a step's `runnable_path` + // is the inner script's, so the scope has to be satisfied through the ancestor. + for (path, expected) in [ + ( + format!("completed/get_result/{FLOW_JOB}"), + r#""flow": "done""#, + ), + ( + format!("completed/get_result/{STEP_JOB}"), + "STEP_RESULT_INHERITED", + ), + ] { + let (status, body) = get(&base, &path, Some("RUN_SCOPED_TOKEN")).await; + assert!( + status.is_success(), + "run-scoped token must read its own flow run ({path}, got {status}): {body}" + ); + assert!( + body.contains(expected), + "run-scoped token should get {expected} for {path}: {body}" + ); + } + // A job of any other runnable is out of scope, even though the same user created it. + for path in [ + format!("completed/get_result/{VICTIM}"), + format!("get_args/{VICTIM}"), + format!("get_logs/{VICTIM}"), + format!("getupdate/{VICTIM}?only_result=true"), + ] { + let (status, body) = get(&base, &path, Some("RUN_SCOPED_TOKEN")).await; + assert_eq!( + status, + reqwest::StatusCode::NOT_FOUND, + "run-scoped token must not read a job outside its scope ({path}, got {status}): {body}" + ); + for secret in [RESULT_SECRET, ARGS_SECRET, LOGS_SECRET] { + assert!( + !body.contains(secret), + "run-scoped token response for {path} leaked `{secret}`: {body}" + ); + } + } + // A `singlestepflow` wrapper (native retry / scheduled run) belongs to the runnable + // it wraps, not to the flow domain its `kind` suggests. Each wrapper is readable by + // the token scoped to the wrapped kind, and only by that one. + for (job, reader, denied) in [ + (WRAPPED_JOB, "RUN_SCOPED_SCRIPT_TOKEN", "RUN_SCOPED_TOKEN"), + ( + WRAPPED_FLOW_JOB, + "RUN_SCOPED_TOKEN", + "RUN_SCOPED_SCRIPT_TOKEN", + ), + ] { + let (status, body) = get(&base, &format!("completed/get_result/{job}"), Some(reader)).await; + assert!( + status.is_success() && body.contains("WRAPPED"), + "{reader} must read the singlestepflow wrapping its runnable (got {status}): {body}" + ); + let (status, body) = get(&base, &format!("completed/get_result/{job}"), Some(denied)).await; + assert_eq!( + status, + reqwest::StatusCode::NOT_FOUND, + "{denied} must not read a wrapper around the other kind (got {status}): {body}" + ); + } + + // An `apps:run:` scope is a start grant too: the inline-script component run it + // launched — a kind no `jobs:run` scope can name — stays readable to a token scoped + // to that app, and stays out of reach for one that is only scoped to run jobs. + let (status, body) = get( + &base, + &format!("completed/get_result/{APP_INLINE_JOB}"), + Some("APP_RUNNER_TOKEN"), + ) + .await; + assert!( + status.is_success() && body.contains("APP_INLINE_RESULT"), + "app-scoped token must read the component run its app launched (got {status}): {body}" + ); + let (status, body) = get( + &base, + &format!("completed/get_result/{APP_INLINE_JOB}"), + Some("RUN_SCOPED_SCRIPT_TOKEN"), + ) + .await; + assert_eq!( + status, + reqwest::StatusCode::NOT_FOUND, + "a token with no scope on the app must not read its component run (got {status}): {body}" + ); + + // An approval link is a bypass of the read gate, so the confinement is re-applied on + // top of it: it must not become a way for a scoped token to read an out-of-scope job. + // The link itself is untouched — a logged-out approver still reads the same job. + let approval_token = + windmill_common::variables::generate_approval_token("test-workspace", VICTIM.parse()?, &db) + .await?; + let (status, body) = get( + &base, + &format!("get/{VICTIM}?approval_token={approval_token}"), + None, + ) + .await; + assert!( + status.is_success(), + "an approval link must still authorize a logged-out read (got {status}): {body}" + ); + let (status, body) = get( + &base, + &format!("get/{VICTIM}?approval_token={approval_token}"), + Some("RUN_SCOPED_TOKEN"), + ) + .await; + assert_eq!( + status, + reqwest::StatusCode::NOT_FOUND, + "an approval link must not lift the run-scope confinement (got {status}): {body}" + ); + + // Same for the resume-secret bypass on the result route, which the approval page uses. + let (status, secret) = get( + &authed_base, + &format!("job_signature/{STEP_JOB}/0"), + Some("SECRET_TOKEN_2"), + ) + .await; + assert!(status.is_success(), "owner must mint a resume secret: {secret}"); + let secret = secret.trim().trim_matches('"').to_string(); + let approval_result = + format!("completed/get_result/{STEP_JOB}?suspended_job={STEP_JOB}&resume_id=0&secret={secret}"); + let (status, body) = get(&base, &approval_result, None).await; + assert!( + status.is_success(), + "a resume secret must still authorize a logged-out result read (got {status}): {body}" + ); + let (status, body) = get(&base, &approval_result, Some("RUN_SCOPED_SCRIPT_TOKEN")).await; + assert_eq!( + status, + reqwest::StatusCode::NOT_FOUND, + "a resume secret must not lift the run-scope confinement (got {status}): {body}" + ); + + // And a run grant is not an enumeration grant: the whole listing surface is denied. + let (status, body) = get(&authed_base, "list", Some("RUN_SCOPED_TOKEN")).await; + assert_eq!( + status, + reqwest::StatusCode::FORBIDDEN, + "run-scoped token must not enumerate jobs (got {status}): {body}" + ); + // ---- APP EMBED TOKEN: cancellation confined to the app's own jobs. The token // may cancel a job it launched (created_by == viewer), but `cancel_job_api` // denies (NotFound) a job created by someone else, even one the (admin) diff --git a/backend/tests/mcp_token_exfil.rs b/backend/tests/mcp_token_exfil.rs index ce278f3c53..1d36ba9546 100644 --- a/backend/tests/mcp_token_exfil.rs +++ b/backend/tests/mcp_token_exfil.rs @@ -19,6 +19,11 @@ //! and the request only fails later at the connect/SSRF step — proving the //! legitimate path still resolves the token (no over-blocking). //! +//! `POST .../resources/mcp_call_tool/{path}` reaches the same MCP server through +//! the same resource, so it is pinned to the same property here — both handlers +//! share `connect_mcp_client`, and a future split of that helper must not let +//! one of them regress. +//! //! SSRF rejection of an author-controlled URL is covered by the unit test in //! `windmill-mcp` (`from_resource_rejects_ssrf_url`). #![cfg(feature = "mcp")] @@ -27,6 +32,7 @@ use sqlx::{Pool, Postgres}; use windmill_test_utils::*; const SECRET_VALUE: &str = "S3CRET-MCP-TOKEN-VALUE"; +const RESOURCE_PATH: &str = "u/test-user-3/evil_mcp"; fn client() -> reqwest::Client { reqwest::Client::new() @@ -44,13 +50,28 @@ async fn get(base: &str, path: &str, token: &str) -> (reqwest::StatusCode, Strin (status, body) } -#[sqlx::test(fixtures("base", "mcp_token_exfil"))] -async fn test_mcp_token_not_exfiltrated(db: Pool) -> anyhow::Result<()> { - initialize_tracing().await; +async fn post( + base: &str, + path: &str, + token: &str, + body: serde_json::Value, +) -> (reqwest::StatusCode, String) { + let resp = client() + .post(format!("{base}/{path}")) + .header("Authorization", format!("Bearer {token}")) + .json(&body) + .send() + .await + .expect("request"); + let status = resp.status(); + let body = resp.text().await.expect("body"); + (status, body) +} - // Insert the locked secret variable with a real, workspace-key-encrypted - // value so an authorized read genuinely decrypts it. - let mc = windmill_common::variables::build_crypt(&db, "test-workspace").await?; +/// Insert the locked secret variable with a real, workspace-key-encrypted value +/// so an authorized read genuinely decrypts it. +async fn insert_locked_secret(db: &Pool) -> anyhow::Result<()> { + let mc = windmill_common::variables::build_crypt(db, "test-workspace").await?; let encrypted = windmill_common::variables::encrypt(&mc, SECRET_VALUE); // Runtime-checked query (not the `query!` macro) so no offline `.sqlx` cache // entry is needed for this test-only insert. @@ -59,13 +80,21 @@ async fn test_mcp_token_not_exfiltrated(db: Pool) -> anyhow::Result<() VALUES ('test-workspace', 'f/locked/secret_token', $1, true, 'Locked secret', '{}')", ) .bind(&encrypted) - .execute(&db) + .execute(db) .await?; + Ok(()) +} + +#[sqlx::test(fixtures("base", "mcp_token_exfil"))] +async fn test_mcp_token_not_exfiltrated(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + insert_locked_secret(&db).await?; let server = ApiServer::start(db.clone()).await?; let port = server.addr.port(); let base = format!("http://localhost:{port}/api/w/test-workspace/resources/mcp_tools"); - let path = "u/test-user-3/evil_mcp"; + let path = RESOURCE_PATH; // ---- CORE REGRESSION: the developer can read the resource but must NOT be // able to resolve the locked secret. They are denied (401) at the @@ -109,3 +138,47 @@ async fn test_mcp_token_not_exfiltrated(db: Pool) -> anyhow::Result<() Ok(()) } + +#[sqlx::test(fixtures("base", "mcp_token_exfil"))] +async fn test_mcp_call_tool_token_not_exfiltrated(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + insert_locked_secret(&db).await?; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/resources/mcp_call_tool"); + let body = serde_json::json!({ "tool": "whoami", "arguments": {} }); + + let (status, resp) = post(&base, RESOURCE_PATH, "SECRET_TOKEN_3", body.clone()).await; + assert_eq!( + status, + reqwest::StatusCode::UNAUTHORIZED, + "developer must be denied resolving a secret they can't read (got {status}): {resp}" + ); + assert!( + !resp.contains(SECRET_VALUE), + "the locked secret must never leak to the developer: {resp}" + ); + assert!( + resp.contains("don't have access"), + "denial should come from the variable-RLS gate, not a connection error: {resp}" + ); + assert!( + !resp.contains("Failed to connect to MCP server"), + "developer must be blocked before the connection step (would mean the token was resolved): {resp}" + ); + + let (status, resp) = post(&base, RESOURCE_PATH, "SECRET_TOKEN", body).await; + assert_ne!( + status, + reqwest::StatusCode::UNAUTHORIZED, + "admin must clear the variable-RLS gate (got {status}): {resp}" + ); + assert!( + resp.contains("Failed to connect to MCP server"), + "admin should resolve the token and only fail at the connect/SSRF step: {resp}" + ); + + Ok(()) +} diff --git a/backend/tests/nativets_stress.rs b/backend/tests/nativets_stress.rs index 32057b41d2..21d3911778 100644 --- a/backend/tests/nativets_stress.rs +++ b/backend/tests/nativets_stress.rs @@ -241,7 +241,6 @@ fn spawn_workers( worker_name, i as u64, n as u32, - "127.0.0.1", rx, tx2, &base_internal_url, diff --git a/backend/tests/worker_ping_ip.rs b/backend/tests/worker_ping_ip.rs new file mode 100644 index 0000000000..8f98398b40 --- /dev/null +++ b/backend/tests/worker_ping_ip.rs @@ -0,0 +1,44 @@ +use sqlx::{Pool, Postgres}; +use windmill_common::{external_ip::UNKNOWN_IP, worker::insert_ping_query}; + +async fn insert_ping(db: &Pool, worker: &str, ip: Option<&str>) -> anyhow::Result<()> { + insert_ping_query( + "test-instance", + worker, + "default", + ip, + &[], + None, + None, + "test", + None, + None, + None, + false, + db, + ) + .await?; + Ok(()) +} + +/// The external IP resolves in the background, so the initial ping often has none yet. That must +/// not blank the address a previous process wrote to the row this one reclaims — worker names are +/// stable across restarts under EXIT_AFTER_N_JOBS. +#[sqlx::test] +async fn unresolved_ip_keeps_the_reclaimed_rows_address(db: Pool) -> anyhow::Result<()> { + insert_ping(&db, "wk-reclaimed", Some("1.2.3.4")).await?; + insert_ping(&db, "wk-reclaimed", None).await?; + let ip: String = sqlx::query_scalar("SELECT ip FROM worker_ping WHERE worker = $1") + .bind("wk-reclaimed") + .fetch_one(&db) + .await?; + assert_eq!(ip, "1.2.3.4"); + + insert_ping(&db, "wk-fresh", None).await?; + let ip: String = sqlx::query_scalar("SELECT ip FROM worker_ping WHERE worker = $1") + .bind("wk-fresh") + .fetch_one(&db) + .await?; + assert_eq!(ip, UNKNOWN_IP); + Ok(()) +} diff --git a/backend/windmill-ai/src/ai_bedrock.rs b/backend/windmill-ai/src/ai_bedrock.rs index 02ab702d31..b223725af2 100644 --- a/backend/windmill-ai/src/ai_bedrock.rs +++ b/backend/windmill-ai/src/ai_bedrock.rs @@ -109,6 +109,19 @@ const BEDROCK_PROMPT_CACHING_SUPPORTED_MODEL_IDS: &[&str] = &[ "anthropic.claude-3-5-sonnet-20241022-v2:0", ]; +/// Claude 4.6 and later are published under several id spellings for the same +/// model (`anthropic.claude-sonnet-4-6`, `...-4-6-v1`, `...-4-6-v1:0`), so they +/// are matched by family prefix rather than by exact id. +const BEDROCK_PROMPT_CACHING_SUPPORTED_MODEL_PREFIXES: &[&str] = &[ + "anthropic.claude-fable-5", + "anthropic.claude-opus-4-6", + "anthropic.claude-opus-4-7", + "anthropic.claude-opus-4-8", + "anthropic.claude-opus-5", + "anthropic.claude-sonnet-4-6", + "anthropic.claude-sonnet-5", +]; + fn build_default_cache_point() -> aws_sdk_bedrockruntime::types::CachePointBlock { aws_sdk_bedrockruntime::types::CachePointBlock::builder() .r#type(aws_sdk_bedrockruntime::types::CachePointType::Default) @@ -123,7 +136,7 @@ fn normalize_bedrock_model_id(model: &str) -> String { .unwrap_or(model) .to_ascii_lowercase(); - for prefix in ["global.", "us.", "eu.", "apac."] { + for prefix in ["global.", "us.", "eu.", "apac.", "au."] { if let Some(normalized_model) = model.strip_prefix(prefix) { return normalized_model.to_string(); } @@ -135,6 +148,9 @@ fn normalize_bedrock_model_id(model: &str) -> String { pub fn bedrock_model_supports_prompt_caching(model: &str) -> bool { let normalized_model = normalize_bedrock_model_id(model); BEDROCK_PROMPT_CACHING_SUPPORTED_MODEL_IDS.contains(&normalized_model.as_str()) + || BEDROCK_PROMPT_CACHING_SUPPORTED_MODEL_PREFIXES + .iter() + .any(|prefix| normalized_model.starts_with(prefix)) } fn append_cache_point_to_system_prompts(system_prompts: &mut Vec) { @@ -1241,6 +1257,27 @@ mod tests { )); } + /// Claude 4.6+ ships under bare, `-v1` and `-v1:0` spellings of the same id, + /// so every one of them has to reach the prefix match. + #[test] + fn bedrock_prompt_caching_supports_claude_4_6_and_later_id_spellings() { + for model in [ + "anthropic.claude-sonnet-4-6", + "anthropic.claude-sonnet-4-6-v1:0", + "us.anthropic.claude-opus-4-6-v1", + "global.anthropic.claude-opus-4-8", + "anthropic.claude-opus-5", + "eu.anthropic.claude-sonnet-5-v1:0", + "au.anthropic.claude-sonnet-5", + "anthropic.claude-fable-5", + ] { + assert!( + bedrock_model_supports_prompt_caching(model), + "{model} must support prompt caching" + ); + } + } + #[test] fn bedrock_prompt_caching_rejects_unsupported_or_opaque_model_ids() { assert!(!bedrock_model_supports_prompt_caching( @@ -1249,5 +1286,9 @@ mod tests { assert!(!bedrock_model_supports_prompt_caching( "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/my-profile" )); + // Opus 4.5 is dated-id only — the 4.6+ prefixes must not swallow it. + assert!(!bedrock_model_supports_prompt_caching( + "anthropic.claude-opus-4-5-20251101-v2:0" + )); } } diff --git a/backend/windmill-ai/src/providers/anthropic.rs b/backend/windmill-ai/src/providers/anthropic.rs index 6ebc555da6..195df62d44 100644 --- a/backend/windmill-ai/src/providers/anthropic.rs +++ b/backend/windmill-ai/src/providers/anthropic.rs @@ -12,6 +12,7 @@ use crate::{ }; use async_trait::async_trait; use http::Method; +use super::REASONING_OFF_SENTINEL; use serde::{Deserialize, Serialize}; use serde_json::value::RawValue; use windmill_common::{client::AuthedClient, error::Error}; @@ -137,17 +138,23 @@ pub struct AnthropicMessage { pub content: Vec, } -/// Adaptive thinking config for Anthropic native API. `summarized` display -/// matches the chat proxy path (renders a summarized thinking stream). +/// Thinking config for the Anthropic native API. `summarized` display matches +/// the chat proxy path (renders a summarized thinking stream); the disable +/// carries no display. #[derive(Serialize, Debug)] pub struct AnthropicThinking { pub r#type: &'static str, - pub display: &'static str, + #[serde(skip_serializing_if = "Option::is_none")] + pub display: Option<&'static str>, } impl AnthropicThinking { fn adaptive() -> Self { - Self { r#type: "adaptive", display: "summarized" } + Self { r#type: "adaptive", display: Some("summarized") } + } + + fn disabled() -> Self { + Self { r#type: "disabled", display: None } } } @@ -157,6 +164,34 @@ pub struct AnthropicOutputConfig { pub effort: String, } +/// Resolve the thinking config, effort and sampling params for a reasoning +/// selection. Adaptive thinking rejects sampling params, so temperature only +/// survives when thinking is off or explicitly disabled (Anthropic returns a +/// hard 400 otherwise). +fn anthropic_thinking_config( + reasoning_effort: Option<&str>, + temperature: Option, +) -> ( + Option, + Option, + Option, +) { + match reasoning_effort { + // The disable sentinel is not an effort token — Anthropic's vocabulary + // is low..max and rejects it. The disable carries no effort either: + // pairing it with xhigh or max is itself a 400 on Opus 5. + Some(effort) if effort == REASONING_OFF_SENTINEL => { + (Some(AnthropicThinking::disabled()), None, temperature) + } + Some(effort) => ( + Some(AnthropicThinking::adaptive()), + Some(AnthropicOutputConfig { effort: effort.to_string() }), + None, + ), + None => (None, None, temperature), + } +} + /// Anthropic-specific request structure for standard API #[derive(Serialize)] pub struct AnthropicRequest<'a> { @@ -652,16 +687,8 @@ impl AnthropicQueryBuilder { } } - // Adaptive thinking rejects sampling params, so drop temperature when - // reasoning is on (Anthropic returns a hard 400 otherwise). - let (thinking, output_config, temperature) = match args.reasoning_effort { - Some(effort) => ( - Some(AnthropicThinking::adaptive()), - Some(AnthropicOutputConfig { effort: effort.to_string() }), - None, - ), - None => (None, None, args.temperature), - }; + let (thinking, output_config, temperature) = + anthropic_thinking_config(args.reasoning_effort, args.temperature); // Build request based on platform if self.is_vertex() { @@ -1096,6 +1123,56 @@ mod tests { assert!(body.get("temperature").is_none()); } + /// An agent step stores the chat's off sentinel verbatim as its + /// `reasoning_effort`, so the disable has to be translated here rather than + /// forwarded as an effort token Anthropic would reject. + #[test] + fn anthropic_thinking_config_translates_the_off_sentinel() { + let (thinking, output_config, temperature) = + anthropic_thinking_config(Some("none"), Some(0.5)); + assert_eq!(thinking.as_ref().map(|t| t.r#type), Some("disabled")); + assert!(output_config.is_none()); + // Sampling params are only rejected alongside adaptive thinking. + assert_eq!(temperature, Some(0.5)); + + let (thinking, output_config, temperature) = + anthropic_thinking_config(Some("xhigh"), Some(0.5)); + assert_eq!(thinking.as_ref().map(|t| t.r#type), Some("adaptive")); + assert_eq!(output_config.map(|c| c.effort), Some("xhigh".to_string())); + assert!(temperature.is_none()); + + let (thinking, output_config, temperature) = anthropic_thinking_config(None, Some(0.5)); + assert!(thinking.is_none()); + assert!(output_config.is_none()); + assert_eq!(temperature, Some(0.5)); + } + + #[test] + fn anthropic_request_serializes_the_off_sentinel_as_a_thinking_disable() { + let request = AnthropicRequest { + model: "claude-opus-5", + system: None, + messages: vec![], + tools: None, + tool_choice: None, + temperature: Some(0.5), + thinking: Some(AnthropicThinking::disabled()), + output_config: None, + max_tokens: Some(64000), + stream: true, + }; + + let body: serde_json::Value = + serde_json::from_str(&serde_json::to_string(&request).unwrap()).unwrap(); + assert_eq!(body["thinking"]["type"], "disabled"); + // A disable paired with an effort is a 400 on Opus 5, and `display` + // only applies to a thinking mode that actually runs. + assert!(body["thinking"].get("display").is_none()); + assert!(body.get("output_config").is_none()); + // Sampling params are only rejected alongside adaptive thinking. + assert_eq!(body["temperature"], 0.5); + } + #[test] fn anthropic_request_omits_thinking_when_reasoning_off() { let request = AnthropicRequest { diff --git a/backend/windmill-ai/src/providers/bedrock.rs b/backend/windmill-ai/src/providers/bedrock.rs index 4405a50dcb..03d460a95f 100644 --- a/backend/windmill-ai/src/providers/bedrock.rs +++ b/backend/windmill-ai/src/providers/bedrock.rs @@ -25,6 +25,7 @@ use crate::{ query_builder::{ParsedResponse, StreamEventSink}, types::{OpenAIMessage, StreamingEvent, TokenUsage, ToolDef}, }; +use super::REASONING_OFF_SENTINEL; use bytes::Bytes; use futures::{stream::BoxStream, StreamExt}; use http::{HeaderMap, Method, StatusCode}; @@ -358,9 +359,7 @@ async fn handle_bedrock_sdk_streaming( let (bedrock_messages, system_prompts) = openai_messages_to_bedrock(&openai_req.messages, enable_prompt_caching)?; // Adaptive thinking rejects sampling params; drop temperature when reasoning is on. - let temperature = openai_req - .reasoning_effort - .is_none() + let temperature = (!effort_enables_thinking(openai_req.reasoning_effort.as_deref())) .then_some(openai_req.temperature) .flatten(); let inference_config = create_inference_config(temperature, openai_req.max_tokens); @@ -410,11 +409,25 @@ async fn handle_bedrock_sdk_streaming( }) } -/// Build the Converse `additionalModelRequestFields` enabling Claude adaptive -/// thinking at the given effort. `display: summarized` is billing-neutral on -/// Anthropic models and matches the direct-Anthropic chat path, which renders -/// summarized thinking in the UI. +/// Whether an effort token turns adaptive thinking on. `"none"` is the disable +/// sentinel rather than a level, and sampling params stay usable alongside it. +fn effort_enables_thinking(effort: Option<&str>) -> bool { + matches!(effort, Some(effort) if effort != REASONING_OFF_SENTINEL) +} + +/// Build the Converse `additionalModelRequestFields` carrying Claude's thinking +/// config. `display: summarized` is billing-neutral on Anthropic models and +/// matches the direct-Anthropic chat path, which renders summarized thinking in +/// the UI. fn bedrock_thinking_fields(effort: &str) -> aws_smithy_types::Document { + if effort == REASONING_OFF_SENTINEL { + // The disable carries no effort: pairing it with xhigh or max is a 400 + // on Opus 5, and omitting it leaves the model at the effort where the + // disable is accepted. + return json_to_document(serde_json::json!({ + "thinking": { "type": "disabled" } + })); + } json_to_document(serde_json::json!({ "thinking": { "type": "adaptive", "display": "summarized" }, "output_config": { "effort": effort } @@ -664,9 +677,7 @@ async fn handle_bedrock_sdk_non_streaming( let (bedrock_messages, system_prompts) = openai_messages_to_bedrock(&openai_req.messages, enable_prompt_caching)?; // Adaptive thinking rejects sampling params; drop temperature when reasoning is on. - let temperature = openai_req - .reasoning_effort - .is_none() + let temperature = (!effort_enables_thinking(openai_req.reasoning_effort.as_deref())) .then_some(openai_req.temperature) .flatten(); let inference_config = create_inference_config(temperature, openai_req.max_tokens); @@ -935,7 +946,8 @@ impl BedrockQueryBuilder { openai_messages_to_bedrock(&prepared_messages, enable_prompt_caching)?; // Adaptive thinking rejects sampling params; drop temperature when reasoning is on. - let temperature = reasoning_effort.is_none().then_some(temperature).flatten(); + let temperature = + (!effort_enables_thinking(reasoning_effort)).then_some(temperature).flatten(); // Build inference configuration using shared helper let inference_config = create_inference_config(temperature, max_tokens.map(|t| t as i32)); @@ -1285,6 +1297,18 @@ mod tests { ); } + #[test] + fn bedrock_thinking_fields_translate_the_off_sentinel_to_a_disable() { + let fields = document_to_json(&bedrock_thinking_fields("none")); + assert_eq!(fields["thinking"]["type"], "disabled"); + // An effort alongside the disable is a 400 on Opus 5. + assert!(fields.get("output_config").is_none()); + // Sampling params survive a disable, unlike adaptive thinking. + assert!(effort_enables_thinking(Some("xhigh"))); + assert!(!effort_enables_thinking(Some("none"))); + assert!(!effort_enables_thinking(None)); + } + #[test] fn bedrock_thinking_fields_carry_adaptive_thinking_and_effort() { let fields = document_to_json(&bedrock_thinking_fields("xhigh")); diff --git a/backend/windmill-ai/src/providers/mod.rs b/backend/windmill-ai/src/providers/mod.rs index 094cc71490..63e4d0de6d 100644 --- a/backend/windmill-ai/src/providers/mod.rs +++ b/backend/windmill-ai/src/providers/mod.rs @@ -8,6 +8,12 @@ pub mod other; use std::time::{Duration, Instant}; +/// The effort token the chat and agent surfaces send to turn reasoning off. +/// It is not a provider-native level — each provider translates it to its own +/// disable (Anthropic and Bedrock to `thinking: {type: "disabled"}`, DeepSeek to +/// its `thinking` param, Gemini to a zero budget or the model's floor). +pub(crate) const REASONING_OFF_SENTINEL: &str = "none"; + use windmill_common::cache::Cache; use crate::{ diff --git a/backend/windmill-api-auth/src/lib.rs b/backend/windmill-api-auth/src/lib.rs index d92c3a5f2d..88eb11093a 100644 --- a/backend/windmill-api-auth/src/lib.rs +++ b/backend/windmill-api-auth/src/lib.rs @@ -67,8 +67,9 @@ pub struct ApiAuthed { /// `label-*` string. Only `username_override_from_label` sets it. pub username_override_is_token_label: bool, /// Whether the request authenticated with the session token minted at browser login. - /// Only `trigger_or_fallback` reads it — see `is_session_label` for why it attributes - /// rather than proves, and must not gate authority. + /// Read by `trigger_or_fallback` and by `TriggerSource::of_request` (which attributes a + /// trigger mutation to the UI) — see `is_session_label` for why it attributes rather than + /// proves, and must not gate authority. pub is_session_token: bool, pub token_prefix: Option, pub read_only: bool, diff --git a/backend/windmill-api-auth/src/scopes.rs b/backend/windmill-api-auth/src/scopes.rs index ed273c7d15..26dd523294 100644 --- a/backend/windmill-api-auth/src/scopes.rs +++ b/backend/windmill-api-auth/src/scopes.rs @@ -274,6 +274,7 @@ pub enum ScopeDomain { // Native trigger domains NativeTriggers, + TriggersHistory, // System domains Audit, @@ -335,6 +336,7 @@ impl ScopeDomain { Self::PostgresTriggers => "postgres_triggers", Self::EmailTriggers => "email_triggers", Self::NativeTriggers => "native_triggers", + Self::TriggersHistory => "triggers_history", Self::Audit => "audit", Self::Settings => "settings", Self::Workers => "workers", @@ -401,6 +403,7 @@ impl ScopeDomain { "indexer" | "srch" => Some(Self::Indexer), "teams" => Some(Self::Teams), "native_triggers" => Some(Self::NativeTriggers), + "triggers_history" => Some(Self::TriggersHistory), "git_sync" | "github_app" => Some(Self::GitSync), "capture" => Some(Self::Capture), "drafts" => Some(Self::Drafts), @@ -689,27 +692,49 @@ fn extract_domain_from_route( ))) } -const RUN_WHITELISTED_GET_PATHS: [&'static str; 20] = [ +/// The reads a `jobs:run` scope implies: following, by id, a run the token started. +/// Every entry is keyed by a job id and confines an authenticated caller to its own +/// runnable — through `require_job_read_access`, through its own `jobs:run:flows:` +/// check, or, where an approval token or resume secret bypasses that gate, through a +/// direct `require_job_within_run_scope`. The one exception is +/// `jobs_u/get_root_job_id/`, which has no check at all but discloses only flow lineage, +/// to anyone, authenticated or not. Workspace-wide enumeration (`jobs/list`, counts, +/// exports) and credential minting (`job_view_token`) are deliberately absent — those are +/// `jobs:read`. Keep by-id read routes here in sync as they are added, or a run token +/// loses the ability to follow its own run through them. +const RUN_WHITELISTED_GET_PATHS: [&'static str; 32] = [ "jobs_u/get_flow/", "jobs_u/get_root_job_id/", "jobs_u/get/", "jobs_u/get_logs/", + "jobs_u/get_completed_logs_tail/", "jobs_u/get_flow_all_logs/", + "jobs_u/get_flow_all_logs_structured/", + "jobs_u/get_flow_all_results/", "jobs_u/get_args/", "jobs_u/get_flow_debug_info/", "jobs_u/completed/get/", "jobs_u/completed/get_result/", "jobs_u/completed/get_result_maybe/", + "jobs_u/completed/get_timing/", + "jobs_u/dispatch_events/", "jobs_u/getupdate/", "jobs_u/getupdate_sse/", "jobs_u/get_log_file/", + "jobs/run_progress/", + "jobs/dbt_graph/", + "jobs/dbt_resumable/", + "jobs/dbt_resumable_script/p/", "jobs/result_by_id/", "jobs/resume_urls/", "jobs/flow/user_states/", "jobs/job_signature/", + "jobs/wac_approval_urls/", "jobs/completed/get/", "jobs/completed/get_result/", "jobs/completed/get_result_maybe/", + "jobs/completed/get_timing/", + "jobs/get_otel_traces/", ]; /// Sentinel scope in app embed tokens. Grants nothing itself; `check_route_access` @@ -826,6 +851,60 @@ fn resource_metadata_route_allowed(suffix: &str) -> bool { || suffix.starts_with("resources/type/") } +/// The `jobs:run` scopes a token's job reads are confined to, or `None` when they are +/// not confined to particular runnables. +/// +/// A run scope is what the trigger UI mints per script or flow and hands to a webhook +/// caller / CI job: it may start the runnables it names and follow those runs, so its +/// by-id job reads must stay within what it can start (enforced by +/// `require_job_read_access`). Both the path (`jobs:run:flows:f/team/etl`) and the +/// kind-only (`jobs:run:scripts`, which legacy `jobs:runscript` tokens carry) forms +/// confine, since `ScopeDefinition::includes` already matches a candidate +/// `jobs:run::` against either. +/// +/// Returns `None` — unconfined — when the token is effectively unscoped, or carries a +/// jobs scope that grants job reads in its own right: `jobs:read`/`jobs:write`, or a +/// bare `jobs:run` (it can start anything, so confining its reads to "what it may run" +/// would restrict nothing). +pub fn job_read_run_confinement(scopes: Option<&[String]>) -> Option> { + let mut confinement = Vec::new(); + for scope in scopes? + .iter() + .filter(|s| !s.starts_with("if_jobs:filter_tags:")) + { + let Ok(scope) = ScopeDefinition::from_scope_string(scope) else { + continue; + }; + if ScopeDomain::from_str(&scope.domain) != Some(ScopeDomain::Jobs) { + continue; + } + match ScopeAction::from_str(&scope.action) { + Some(ScopeAction::Run) if scope.kind.is_some() || scope.resource.is_some() => { + confinement.push(scope) + } + Some(_) => return None, + None => continue, + } + } + (!confinement.is_empty()).then_some(confinement) +} + +/// Whether a job that ran `runnable_path` as `kind` (`scripts` or `flows`) is inside a +/// [`job_read_run_confinement`] set. +pub fn run_confinement_admits( + confinement: &[ScopeDefinition], + kind: &str, + runnable_path: &str, +) -> bool { + let required = ScopeDefinition::new( + ScopeDomain::Jobs.as_str(), + ScopeAction::Run.as_str(), + Some(kind), + Some(vec![runnable_path.to_string()]), + ); + confinement.iter().any(|scope| scope.includes(&required)) +} + fn scope_grants_access( scope: &ScopeDefinition, required_domain: ScopeDomain, @@ -862,15 +941,26 @@ fn scope_grants_access( return Ok(true); } - if !scope_action.includes(&required_action) - && !(scope_domain == ScopeDomain::Jobs - && required_action == ScopeAction::Read - && route_path.is_some_and(|p| { - RUN_WHITELISTED_GET_PATHS - .iter() - .any(|path| p.starts_with(path)) - })) + // `jobs:run` is a grant to *start* a runnable. The only reads it implies are the + // by-id routes a caller needs to follow the run it started + // (`RUN_WHITELISTED_GET_PATHS`) — never workspace-wide enumeration (`jobs/list`, + // counts, exports), which is what `jobs:read` is for. Those by-id reads are in turn + // confined to the runnable a path-scoped token names, by `require_job_read_access`. + // `ScopeAction::Run.includes(&Read)` (which exists so `apps:run` can fetch the app + // it runs) must not reach this domain, so decide it here rather than falling + // through to the hierarchy below. + if scope_domain == ScopeDomain::Jobs + && scope_action == ScopeAction::Run + && required_action == ScopeAction::Read { + return Ok(route_path.is_some_and(|p| { + RUN_WHITELISTED_GET_PATHS + .iter() + .any(|path| p.starts_with(path)) + })); + } + + if !scope_action.includes(&required_action) { return Ok(false); } @@ -1099,6 +1189,65 @@ mod tests { .is_err()); } + #[test] + fn jobs_run_reads_are_limited_to_the_by_id_poll_routes() { + let job = "/api/w/test/jobs_u/completed/get_result/019ff012-6b1e-0d6b-fc0d-0c85d34d9cec"; + let list = "/api/w/test/jobs/list"; + for scope in ["jobs:run", "jobs:run:scripts:u/admin/script"] { + // Following the run it started stays available... + assert!( + check_route_access(&[scope.to_string()], job, "GET").is_ok(), + "{scope} must reach the by-id job poll routes" + ); + // ...but a run grant is not a licence to enumerate the workspace's jobs. + assert!( + check_route_access(&[scope.to_string()], list, "GET").is_err(), + "{scope} must not reach jobs/list" + ); + } + assert!(check_route_access(&["jobs:read".to_string()], list, "GET").is_ok()); + } + + #[test] + fn run_scopes_confine_job_reads_by_kind_and_path() { + let confinement = + job_read_run_confinement(Some(&["jobs:run:flows:f/team/*".to_string()])).unwrap(); + assert!(run_confinement_admits(&confinement, "flows", "f/team/etl")); + // Right path, wrong kind — a script named like the flow is not the flow. + assert!(!run_confinement_admits( + &confinement, + "scripts", + "f/team/etl" + )); + assert!(!run_confinement_admits( + &confinement, + "flows", + "f/other/etl" + )); + + // A kind-only scope confines to that kind, at any path. + let kind_only = job_read_run_confinement(Some(&["jobs:run:scripts".to_string()])).unwrap(); + assert!(run_confinement_admits(&kind_only, "scripts", "u/admin/anything")); + assert!(!run_confinement_admits(&kind_only, "flows", "f/team/etl")); + + // Scopes that grant job reads in their own right leave reads unconfined. + for scopes in [ + vec!["jobs:read".to_string()], + vec!["jobs:run".to_string()], + vec![ + "jobs:run:scripts:u/admin/script".to_string(), + "jobs:read".to_string(), + ], + vec!["if_jobs:filter_tags:deno".to_string()], + ] { + assert!( + job_read_run_confinement(Some(&scopes)).is_none(), + "{scopes:?} must not confine job reads" + ); + } + assert!(job_read_run_confinement(None).is_none()); + } + #[test] fn test_new_domain_parsing() { // Test that new domains are properly parsed diff --git a/backend/windmill-api-groups/src/groups.rs b/backend/windmill-api-groups/src/groups.rs index d2a6c7d16e..69b50e4b83 100644 --- a/backend/windmill-api-groups/src/groups.rs +++ b/backend/windmill-api-groups/src/groups.rs @@ -514,6 +514,143 @@ async fn update_igroup( Ok(format!("Updated group {}", name)) } +/// Workspaces whose auto-assignment config references any of `groups`. +/// +/// Reads `workspace_settings` across the whole instance without checking the caller's rights. +/// Callers must have established superadmin beforehand; the result leaks which workspaces are +/// configured with a given instance group. +/// +/// This and every reconcile call site are gated on `private` alone, NOT `enterprise`: CE +/// builds ship `private` without `enterprise`, and gating on `enterprise` would scrub +/// references while stranding the affected workspace members on CE. +#[cfg(feature = "private")] +pub async fn workspaces_referencing_instance_groups( + groups: &[String], + tx: &mut Transaction<'_, Postgres>, +) -> Result> { + if groups.is_empty() { + return Ok(vec![]); + } + + let workspaces = sqlx::query_scalar!( + "SELECT workspace_id FROM workspace_settings WHERE auto_invite->'instance_groups' ?| $1", + groups + ) + .fetch_all(&mut **tx) + .await?; + + Ok(workspaces) +} + +/// Compute and advisory-lock every workspace whose auto-assignment config references any of +/// `groups`. Mutation paths call this after locking their `instance_group` rows and before +/// any other row lock — the hierarchy is group rows → workspace advisory locks → all other +/// row locks (see `reconcile_workspace_instance_groups`). Same authorization contract as +/// `workspaces_referencing_instance_groups`. +#[cfg(feature = "private")] +pub async fn lock_workspaces_referencing_instance_groups( + groups: &[String], + tx: &mut Transaction<'_, Postgres>, +) -> Result> { + use windmill_api_workspaces::workspaces_ee::lock_instance_group_workspaces; + + let workspaces = workspaces_referencing_instance_groups(groups, tx).await?; + lock_instance_group_workspaces(&workspaces, tx).await?; + Ok(workspaces) +} + +/// Drop `groups` from every workspace's instance-group auto-assignment config. +/// +/// Workspaces reference instance groups by name in `workspace_settings.auto_invite`, and +/// nothing in the schema ties those references to `instance_group` rows. A deleted group whose +/// name is left behind here silently re-acquires its members if a group of the same name is +/// created later. +/// +/// Mutates every workspace's settings, so callers must have established superadmin first. +/// Deliberately not audited per workspace: the mutation is instance-scoped and recorded by +/// the caller's global igroup audit event. +pub async fn remove_instance_groups_from_workspace_settings( + groups: &[String], + tx: &mut Transaction<'_, Postgres>, +) -> Result<()> { + if groups.is_empty() { + return Ok(()); + } + + // Row filter must stay `?|`: it yields false on a JSON `null` instance_groups, where + // jsonb_array_elements_text would instead raise and abort the whole transaction; the + // jsonb_typeof guard rules out the same class of value for the roles object. The filter is + // not index-backed — the GIN index covers the auto_invite column, not this expression — + // which is acceptable since workspace_settings holds one row per workspace. + sqlx::query!( + r#"UPDATE workspace_settings SET + auto_invite = jsonb_set( + jsonb_set( + COALESCE(auto_invite, '{}'::jsonb), + '{instance_groups}', + (SELECT COALESCE(jsonb_agg(elem), '[]'::jsonb) + FROM jsonb_array_elements(COALESCE(auto_invite->'instance_groups', '[]'::jsonb)) elem + WHERE elem #>> '{}' <> ALL($1)) + ), + '{instance_groups_roles}', + CASE WHEN jsonb_typeof(auto_invite->'instance_groups_roles') = 'object' + THEN (auto_invite->'instance_groups_roles') - $1::text[] + ELSE '{}'::jsonb + END + ) + WHERE auto_invite->'instance_groups' ?| $1"#, + groups + ) + .execute(&mut **tx) + .await?; + + Ok(()) +} + +/// Follow an instance-group rename through every workspace's auto-assignment config. +/// +/// Workspaces reference instance groups by name, so a rename that leaves the old name behind +/// strands those references: the reconciler resolves membership from the groups a workspace +/// references, and a name that no longer matches any group reads as "no members", which would +/// evict everyone granted through it on the next reconcile. +/// +/// Mutates every workspace's settings, so callers must have established superadmin first. +/// Deliberately not audited per workspace: the mutation is instance-scoped and recorded by +/// the caller's global igroup audit event. +pub async fn rename_instance_group_in_workspace_settings( + old_name: &str, + new_name: &str, + tx: &mut Transaction<'_, Postgres>, +) -> Result<()> { + // Row filter must stay `?`: it yields false on a JSON `null` instance_groups, where + // jsonb_array_elements would instead raise and abort the whole transaction. + sqlx::query!( + r#"UPDATE workspace_settings SET + auto_invite = jsonb_set( + jsonb_set( + COALESCE(auto_invite, '{}'::jsonb), + '{instance_groups}', + (SELECT COALESCE(jsonb_agg( + CASE WHEN elem #>> '{}' = $1 THEN to_jsonb($2::text) ELSE elem END), '[]'::jsonb) + FROM jsonb_array_elements(COALESCE(auto_invite->'instance_groups', '[]'::jsonb)) elem) + ), + '{instance_groups_roles}', + CASE WHEN COALESCE(auto_invite->'instance_groups_roles', '{}'::jsonb) ? $1 + THEN (COALESCE(auto_invite->'instance_groups_roles', '{}'::jsonb) - $1) + || jsonb_build_object($2::text, auto_invite->'instance_groups_roles'->$1) + ELSE COALESCE(auto_invite->'instance_groups_roles', '{}'::jsonb) + END + ) + WHERE auto_invite->'instance_groups' ? $1"#, + old_name, + new_name + ) + .execute(&mut **tx) + .await?; + + Ok(()) +} + async fn delete_igroup( authed: ApiAuthed, Extension(db): Extension, @@ -522,9 +659,10 @@ async fn delete_igroup( require_super_admin(&db, &authed.email).await?; let mut tx: Transaction<'_, Postgres> = db.begin().await?; - // Fetch group's instance_role and members before deletion + // FOR UPDATE: the group row is the group-level mutex, taken before the workspace + // advisory locks (see reconcile_workspace_instance_groups). let group_role = sqlx::query_scalar!( - "SELECT instance_role FROM instance_group WHERE name = $1", + "SELECT instance_role FROM instance_group WHERE name = $1 FOR UPDATE", &name ) .fetch_optional(&mut *tx) @@ -539,6 +677,13 @@ async fn delete_igroup( vec![] }; + // Captured and advisory-locked before the settings update strips the group from them. + #[cfg(feature = "private")] + let affected_workspaces = + lock_workspaces_referencing_instance_groups(std::slice::from_ref(&name), &mut tx).await?; + + remove_instance_groups_from_workspace_settings(std::slice::from_ref(&name), &mut tx).await?; + sqlx::query!("DELETE FROM email_to_igroup WHERE igroup = $1", name) .execute(&mut *tx) .await?; @@ -553,6 +698,12 @@ async fn delete_igroup( apply_instance_role(email, effective_role.as_deref(), &mut tx).await?; } + #[cfg(feature = "private")] + { + use windmill_api_workspaces::workspaces_ee::reconcile_workspace_instance_groups; + reconcile_workspace_instance_groups(&affected_workspaces, &mut tx, &authed).await?; + } + audit_log( &mut *tx, &authed, @@ -823,12 +974,22 @@ async fn add_user_igroup( let mut tx: Transaction<'_, Postgres> = db.begin().await?; - let group_opt = sqlx::query_scalar!("SELECT name FROM instance_group WHERE name = $1", name) - .fetch_optional(&mut *tx) - .await?; + // FOR UPDATE: the group row is the group-level mutex, taken before the workspace + // advisory locks (see reconcile_workspace_instance_groups). + let group_opt = sqlx::query_scalar!( + "SELECT name FROM instance_group WHERE name = $1 FOR UPDATE", + name + ) + .fetch_optional(&mut *tx) + .await?; not_found_if_none(group_opt, "IGroup", &name)?; + // Before the membership insert's row lock. + #[cfg(feature = "private")] + let affected_workspaces = + lock_workspaces_referencing_instance_groups(std::slice::from_ref(&name), &mut tx).await?; + sqlx::query!( "INSERT INTO email_to_igroup (email, igroup) VALUES ($1, $2) ON CONFLICT DO NOTHING", email, @@ -848,86 +1009,17 @@ async fn add_user_igroup( ) .await?; - // Sync user to workspaces configured with this instance group - #[cfg(all(feature = "private", feature = "enterprise"))] - { - use windmill_api_workspaces::workspaces_ee::auto_add_user; - use windmill_common::users::compute_highest_workspace_role; - - // Find all instance groups this user belongs to (includes the newly added group) - let user_igroups: Vec = sqlx::query_scalar!( - "SELECT igroup FROM email_to_igroup WHERE email = $1", - &email - ) - .fetch_all(&mut *tx) - .await?; - - let workspaces = sqlx::query!( - r#" - SELECT workspace_id, - auto_invite->'instance_groups_roles' as instance_groups_roles, - auto_invite->'instance_groups' as instance_groups_json - FROM workspace_settings - WHERE auto_invite->'instance_groups' ? $1 - "#, - &name - ) - .fetch_all(&mut *tx) - .await?; - - for ws in workspaces { - let roles: std::collections::HashMap = ws - .instance_groups_roles - .and_then(|r| serde_json::from_value(r).ok()) - .unwrap_or_default(); - - let ws_configured_groups: Vec = ws - .instance_groups_json - .and_then(|ig| serde_json::from_value(ig).ok()) - .unwrap_or_default(); - - let (best_group, is_admin, is_operator) = - compute_highest_workspace_role(&user_igroups, &ws_configured_groups, &roles); - - let instance_group_source = serde_json::json!({ - "source": "instance_group", - "group": &best_group - }); - - // auto_add_user creates the user if they don't exist (ON CONFLICT DO NOTHING). - // The operator flag here doesn't matter for the final state — the UPDATE below - // always sets the correct is_admin/operator based on the highest-precedence role. - auto_add_user( - &email, - &ws.workspace_id, - &false, - &mut tx, - &authed, - Some(instance_group_source.clone()), - ) - .await?; - - // Set the correct role based on highest precedence across all groups. - // For new users, auto_add_user already stored added_via with source=instance_group, - // so this UPDATE will match. For existing instance_group users, it upgrades/corrects - // the role. Manually-added users (added_via is NULL or non-instance_group) are not affected. - sqlx::query!( - "UPDATE usr SET is_admin = $1, operator = $2, added_via = $3 WHERE workspace_id = $4 AND email = $5 AND added_via->>'source' = 'instance_group'", - is_admin, - is_operator, - &instance_group_source, - &ws.workspace_id, - &email - ) - .execute(&mut *tx) - .await?; - } - } - // Apply instance-level role from group membership let effective_role = compute_effective_instance_role(&email, &mut tx).await?; apply_instance_role(&email, effective_role.as_deref(), &mut tx).await?; + // Sync workspace membership derived from this instance group. + #[cfg(feature = "private")] + { + use windmill_api_workspaces::workspaces_ee::reconcile_workspace_instance_groups; + reconcile_workspace_instance_groups(&affected_workspaces, &mut tx, &authed).await?; + } + tx.commit().await?; Ok(format!("Added {} to igroup {}", email, name)) } @@ -1100,12 +1192,22 @@ async fn remove_user_igroup( require_super_admin(&db, &authed.email).await?; let mut tx = db.begin().await?; - let group_opt = sqlx::query_scalar!("SELECT name FROM instance_group WHERE name = $1", name,) - .fetch_optional(&mut *tx) - .await?; + // FOR UPDATE: the group row is the group-level mutex, taken before the workspace + // advisory locks (see reconcile_workspace_instance_groups). + let group_opt = sqlx::query_scalar!( + "SELECT name FROM instance_group WHERE name = $1 FOR UPDATE", + name, + ) + .fetch_optional(&mut *tx) + .await?; not_found_if_none(group_opt, "IGroup", &name)?; + // Before the membership delete's row lock. + #[cfg(feature = "private")] + let affected_workspaces = + lock_workspaces_referencing_instance_groups(std::slice::from_ref(&name), &mut tx).await?; + sqlx::query!( "DELETE FROM email_to_igroup WHERE email = $1 AND igroup = $2", email, @@ -1125,17 +1227,19 @@ async fn remove_user_igroup( ) .await?; - // Remove user from workspaces where they were added via this instance group - #[cfg(all(feature = "private", feature = "enterprise"))] - { - use windmill_api_workspaces::workspaces_ee::remove_users_from_instance_group_workspaces; - remove_users_from_instance_group_workspaces(&email, &name, &mut tx).await?; - } - // Recompute instance-level role after group removal let effective_role = compute_effective_instance_role(&email, &mut tx).await?; apply_instance_role(&email, effective_role.as_deref(), &mut tx).await?; + // Re-derive workspace membership now that the base tables reflect the removal: drops the + // user where this group was their only access source, or re-roles them from the groups + // they still belong to. + #[cfg(feature = "private")] + { + use windmill_api_workspaces::workspaces_ee::reconcile_workspace_instance_groups; + reconcile_workspace_instance_groups(&affected_workspaces, &mut tx, &authed).await?; + } + tx.commit().await?; Ok(format!("Removed {} from igroup {}", email, name)) } @@ -1265,6 +1369,37 @@ async fn overwrite_igroups( require_super_admin(&db, &authed.email).await?; let mut tx = db.begin().await?; + // The import replaces the whole group catalog, so the whole-table lock is its + // group-mutex phase, taken first like every path's group locks (see + // reconcile_workspace_instance_groups). Per-row FOR UPDATE would miss rows committed + // after the scan, which the unqualified deletes below would then lock after the + // workspace locks — the inverted order. EXCLUSIVE conflicts with the writes and the + // FOR UPDATE of every other mutation path while leaving plain reads unblocked. + sqlx::query("LOCK TABLE instance_group IN EXCLUSIVE MODE") + .execute(&mut *tx) + .await?; + + let imported_names: Vec = igroups.iter().map(|g| g.name.clone()).collect(); + // NULL-safe and correct for an empty import: `name <> ALL('{}')` is true for every row. + let previous_names: Vec = sqlx::query_scalar!( + "SELECT name FROM instance_group WHERE name <> ALL($1)", + &imported_names + ) + .fetch_all(&mut *tx) + .await?; + + // Membership of retained groups is wiped and re-imported below, so workspaces referencing + // either side of the import may see their projection change. Captured and advisory-locked + // before the settings update strips the dropped groups from them. + #[cfg(feature = "private")] + let affected_workspaces = { + let mut all_names = previous_names.clone(); + all_names.extend(imported_names.iter().cloned()); + lock_workspaces_referencing_instance_groups(&all_names, &mut tx).await? + }; + + remove_instance_groups_from_workspace_settings(&previous_names, &mut tx).await?; + sqlx::query!("DELETE FROM email_to_igroup") .execute(&mut *tx) .await?; @@ -1325,6 +1460,15 @@ async fn overwrite_igroups( apply_instance_role(email, None, &mut tx).await?; } + // Runs after the re-insert so the reconciler judges membership against the imported + // state: a member who moved from a dropped group to a retained one is re-roled in place + // instead of losing workspace access. + #[cfg(feature = "private")] + { + use windmill_api_workspaces::workspaces_ee::reconcile_workspace_instance_groups; + reconcile_workspace_instance_groups(&affected_workspaces, &mut tx, &authed).await?; + } + audit_log( &mut *tx, &authed, diff --git a/backend/windmill-api-integration-tests/tests/groups.rs b/backend/windmill-api-integration-tests/tests/groups.rs index 0fd8ee7e08..f86522aae3 100644 --- a/backend/windmill-api-integration-tests/tests/groups.rs +++ b/backend/windmill-api-integration-tests/tests/groups.rs @@ -159,12 +159,7 @@ async fn test_group_endpoints(db: Pool) -> anyhow::Result<()> { .send() .await .unwrap(); - assert_eq!( - resp.status(), - 200, - "create igroup: {}", - resp.text().await? - ); + assert_eq!(resp.status(), 200, "create igroup: {}", resp.text().await?); // --- list instance groups --- let resp = authed(client().get(format!("{global_base}/list"))) @@ -199,12 +194,7 @@ async fn test_group_endpoints(db: Pool) -> anyhow::Result<()> { .send() .await .unwrap(); - assert_eq!( - resp.status(), - 200, - "update igroup: {}", - resp.text().await? - ); + assert_eq!(resp.status(), 200, "update igroup: {}", resp.text().await?); // verify update let resp = authed(client().get(format!("{global_base}/get/test_igroup"))) @@ -220,12 +210,7 @@ async fn test_group_endpoints(db: Pool) -> anyhow::Result<()> { .send() .await .unwrap(); - assert_eq!( - resp.status(), - 200, - "adduser igroup: {}", - resp.text().await? - ); + assert_eq!(resp.status(), 200, "adduser igroup: {}", resp.text().await?); // verify membership let resp = authed(client().get(format!("{global_base}/get/test_igroup"))) @@ -243,13 +228,11 @@ async fn test_group_endpoints(db: Pool) -> anyhow::Result<()> { ); // --- removeuser from instance group --- - let resp = authed(client().post(format!( - "{global_base}/removeuser/test_igroup" - ))) - .json(&json!({"email": "test@windmill.dev"})) - .send() - .await - .unwrap(); + let resp = authed(client().post(format!("{global_base}/removeuser/test_igroup"))) + .json(&json!({"email": "test@windmill.dev"})) + .send() + .await + .unwrap(); assert_eq!(resp.status(), 200); // --- export (EE-gated) --- @@ -280,12 +263,7 @@ async fn test_group_endpoints(db: Pool) -> anyhow::Result<()> { .send() .await .unwrap(); - assert_eq!( - resp.status(), - 200, - "delete igroup: {}", - resp.text().await? - ); + assert_eq!(resp.status(), 200, "delete igroup: {}", resp.text().await?); // verify deleted let resp = authed(client().get(format!("{global_base}/list"))) @@ -297,3 +275,641 @@ async fn test_group_endpoints(db: Pool) -> anyhow::Result<()> { Ok(()) } + +/// Deleting an instance group must not revoke workspace access a member still holds through +/// another configured group. +/// +/// `added_via.group` records only the member's highest-precedence group, so any cleanup keyed +/// on that field alone evicts members who still qualify via a lower-precedence one. Membership +/// must be re-derived from all the groups the workspace still references. +#[cfg(feature = "private")] +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_delete_instance_group_preserves_access_via_other_group( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let global_base = format!("http://localhost:{port}/api/groups"); + let ws_base = format!("http://localhost:{port}/api/w/test-workspace/workspaces"); + + for g in ["igroup_a", "igroup_b"] { + let resp = authed(client().post(format!("{global_base}/create"))) + .json(&json!({ "name": g, "summary": g })) + .send() + .await?; + assert_eq!(resp.status(), 200, "create {g}"); + } + + // multi@ belongs to both groups; only_a@ only to the group that gets deleted. + for (g, email) in [ + ("igroup_a", "multi@example.com"), + ("igroup_b", "multi@example.com"), + ("igroup_a", "only_a@example.com"), + ] { + let resp = authed(client().post(format!("{global_base}/adduser/{g}"))) + .json(&json!({ "email": email })) + .send() + .await?; + assert_eq!(resp.status(), 200, "adduser {g}/{email}"); + } + + // igroup_a grants the higher-precedence role, so added_via lands on it. + let resp = authed(client().post(format!("{ws_base}/edit_instance_groups"))) + .json(&json!({ + "groups": ["igroup_a", "igroup_b"], + "roles": { "igroup_a": "admin", "igroup_b": "developer" } + })) + .send() + .await?; + assert_eq!( + resp.status(), + 200, + "edit_instance_groups: {}", + resp.text().await? + ); + + let (is_admin, via): (bool, Option) = sqlx::query_as( + "SELECT is_admin, added_via->>'group' FROM usr + WHERE workspace_id = 'test-workspace' AND email = 'multi@example.com'", + ) + .fetch_one(&db) + .await?; + assert!(is_admin, "multi@ should start as admin via igroup_a"); + assert_eq!(via.as_deref(), Some("igroup_a")); + + // Workspace state that must survive the group removal. `delete_workspace_user_internal` + // drops all of this, so a delete-and-re-add of a still-qualifying member loses it silently. + let username: String = sqlx::query_scalar( + "SELECT username FROM usr WHERE workspace_id = 'test-workspace' AND email = 'multi@example.com'", + ) + .fetch_one(&db) + .await?; + sqlx::query( + "INSERT INTO favorite (workspace_id, usr, path, favorite_kind) + VALUES ('test-workspace', $1, 'f/keep/me', 'script')", + ) + .bind(&username) + .execute(&db) + .await?; + sqlx::query( + "INSERT INTO draft (workspace_id, path, typ, value) + VALUES ('test-workspace', 'u/' || $1 || '/keep', 'script', '{}'::jsonb)", + ) + .bind(&username) + .execute(&db) + .await?; + + let resp = authed(client().delete(format!("{global_base}/delete/igroup_a"))) + .send() + .await?; + assert_eq!( + resp.status(), + 200, + "delete igroup_a: {}", + resp.text().await? + ); + + // Still a member, downgraded to igroup_b's role rather than evicted. + let (is_admin, is_operator, via): (bool, bool, Option) = sqlx::query_as( + "SELECT is_admin, operator, added_via->>'group' FROM usr + WHERE workspace_id = 'test-workspace' AND email = 'multi@example.com'", + ) + .fetch_one(&db) + .await?; + assert!(!is_admin, "multi@ should lose admin with igroup_a gone"); + assert!(!is_operator, "igroup_b grants developer, not operator"); + assert_eq!( + via.as_deref(), + Some("igroup_b"), + "added_via should re-point at the surviving group" + ); + + // Their workspace state is intact: they were never deleted and re-added. + let favorites: i64 = sqlx::query_scalar( + "SELECT count(*) FROM favorite WHERE workspace_id = 'test-workspace' AND path = 'f/keep/me'", + ) + .fetch_one(&db) + .await?; + assert_eq!( + favorites, 1, + "favorite must survive losing a non-sole group" + ); + let drafts: i64 = sqlx::query_scalar( + "SELECT count(*) FROM draft WHERE workspace_id = 'test-workspace' AND path LIKE 'u/%/keep'", + ) + .fetch_one(&db) + .await?; + assert_eq!(drafts, 1, "draft must survive losing a non-sole group"); + + // igroup_a was only_a@'s sole path in, so they are removed. + let remaining: i64 = sqlx::query_scalar( + "SELECT count(*) FROM usr + WHERE workspace_id = 'test-workspace' AND email = 'only_a@example.com'", + ) + .fetch_one(&db) + .await?; + assert_eq!(remaining, 0, "only_a@ should be removed with igroup_a"); + + // The deleted group leaves no dangling reference in either auto_invite field. + let (groups, roles): (serde_json::Value, serde_json::Value) = sqlx::query_as( + "SELECT auto_invite->'instance_groups', auto_invite->'instance_groups_roles' + FROM workspace_settings WHERE workspace_id = 'test-workspace'", + ) + .fetch_one(&db) + .await?; + assert_eq!(groups, json!(["igroup_b"]), "igroup_a should be stripped"); + assert_eq!( + roles, + json!({ "igroup_b": "developer" }), + "igroup_a's role entry should be stripped" + ); + + Ok(()) +} + +/// Removing a member from one instance group must re-derive their role from the groups they +/// still belong to, not leave the privileges the removed group granted. +/// +/// Still-qualifying members keep their `usr` row (deleting it would destroy their workspace +/// data), so the removal path must recompute that row's role — otherwise a member dropped +/// from an admin group keeps `is_admin` through the stale row. +#[cfg(feature = "private")] +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_remove_user_from_instance_group_rederives_role( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let global_base = format!("http://localhost:{port}/api/groups"); + let ws_base = format!("http://localhost:{port}/api/w/test-workspace/workspaces"); + + for g in ["role_a", "role_b"] { + let resp = authed(client().post(format!("{global_base}/create"))) + .json(&json!({ "name": g })) + .send() + .await?; + assert_eq!(resp.status(), 200, "create {g}"); + let resp = authed(client().post(format!("{global_base}/adduser/{g}"))) + .json(&json!({ "email": "demoted@example.com" })) + .send() + .await?; + assert_eq!(resp.status(), 200, "adduser {g}"); + } + + let resp = authed(client().post(format!("{ws_base}/edit_instance_groups"))) + .json(&json!({ + "groups": ["role_a", "role_b"], + "roles": { "role_a": "admin", "role_b": "developer" } + })) + .send() + .await?; + assert_eq!(resp.status(), 200, "edit: {}", resp.text().await?); + + let is_admin: bool = sqlx::query_scalar( + "SELECT is_admin FROM usr WHERE workspace_id = 'test-workspace' AND email = 'demoted@example.com'", + ) + .fetch_one(&db) + .await?; + assert!(is_admin, "should start admin via role_a"); + + // Drop them from the admin group only. + let resp = authed(client().post(format!("{global_base}/removeuser/role_a"))) + .json(&json!({ "email": "demoted@example.com" })) + .send() + .await?; + assert_eq!(resp.status(), 200, "removeuser: {}", resp.text().await?); + + let (is_admin, is_operator, via): (bool, bool, Option) = sqlx::query_as( + "SELECT is_admin, operator, added_via->>'group' FROM usr + WHERE workspace_id = 'test-workspace' AND email = 'demoted@example.com'", + ) + .fetch_one(&db) + .await?; + assert!( + !is_admin, + "admin granted by role_a must not survive removal from role_a" + ); + assert!(!is_operator, "role_b grants developer"); + assert_eq!(via.as_deref(), Some("role_b")); + + Ok(()) +} + +/// An overwrite import that moves a member from a dropped group to a retained one must keep +/// their workspace data. +/// +/// Qualification must be judged against the imported membership, not the pre-import state: +/// judged too early, the member's new group is not yet visible, they are deleted, and any +/// re-add creates a fresh row stripped of everything workspace-scoped. +#[cfg(all(feature = "private", feature = "enterprise"))] +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_overwrite_igroups_preserves_moved_member_data( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let global_base = format!("http://localhost:{port}/api/groups"); + let ws_base = format!("http://localhost:{port}/api/w/test-workspace/workspaces"); + + for g in ["move_from", "move_to"] { + let resp = authed(client().post(format!("{global_base}/create"))) + .json(&json!({ "name": g })) + .send() + .await?; + assert_eq!(resp.status(), 200, "create {g}"); + } + // Member starts only in move_from. + let resp = authed(client().post(format!("{global_base}/adduser/move_from"))) + .json(&json!({ "email": "mover@example.com" })) + .send() + .await?; + assert_eq!(resp.status(), 200); + + let resp = authed(client().post(format!("{ws_base}/edit_instance_groups"))) + .json(&json!({ + "groups": ["move_from", "move_to"], + "roles": { "move_from": "developer", "move_to": "developer" } + })) + .send() + .await?; + assert_eq!(resp.status(), 200, "edit: {}", resp.text().await?); + + let username: String = sqlx::query_scalar( + "SELECT username FROM usr WHERE workspace_id = 'test-workspace' AND email = 'mover@example.com'", + ) + .fetch_one(&db) + .await?; + sqlx::query( + "INSERT INTO favorite (workspace_id, usr, path, favorite_kind) + VALUES ('test-workspace', $1, 'f/moved/keep', 'script')", + ) + .bind(&username) + .execute(&db) + .await?; + + // Import drops move_from entirely and puts the member in move_to instead. + let resp = authed(client().post(format!("{global_base}/overwrite"))) + .json(&json!([ + { "name": "move_to", "emails": ["mover@example.com"] } + ])) + .send() + .await?; + assert_eq!(resp.status(), 200, "overwrite: {}", resp.text().await?); + + let remaining: i64 = sqlx::query_scalar( + "SELECT count(*) FROM usr WHERE workspace_id = 'test-workspace' AND email = 'mover@example.com'", + ) + .fetch_one(&db) + .await?; + assert_eq!( + remaining, 1, + "member should still be in the workspace via move_to" + ); + + let favorites: i64 = sqlx::query_scalar( + "SELECT count(*) FROM favorite WHERE workspace_id = 'test-workspace' AND path = 'f/moved/keep'", + ) + .fetch_one(&db) + .await?; + assert_eq!( + favorites, 1, + "moving between groups in one import must not destroy workspace data" + ); + + Ok(()) +} + +/// A full-import overwrite must reconcile the membership of retained groups too: a member +/// dropped from a retained group loses the access that group granted, and a member who only +/// lost their highest-precedence group is re-roled in place instead of keeping a stale +/// elevated role. +/// +/// Regression: the delta-based cleanup only acted on groups that disappeared from the import, +/// so an import that kept a group but dropped some of its members never cleaned those members +/// up. +#[cfg(all(feature = "private", feature = "enterprise"))] +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_overwrite_igroups_reconciles_retained_group_membership( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let global_base = format!("http://localhost:{port}/api/groups"); + let ws_base = format!("http://localhost:{port}/api/w/test-workspace/workspaces"); + + for g in ["top_admins", "base_devs"] { + let resp = authed(client().post(format!("{global_base}/create"))) + .json(&json!({ "name": g })) + .send() + .await?; + assert_eq!(resp.status(), 200, "create {g}"); + } + + // demoted@ holds admin via top_admins and developer via base_devs; dropped@ only has + // base_devs. + for (g, email) in [ + ("top_admins", "demoted@example.com"), + ("base_devs", "demoted@example.com"), + ("base_devs", "dropped@example.com"), + ] { + let resp = authed(client().post(format!("{global_base}/adduser/{g}"))) + .json(&json!({ "email": email })) + .send() + .await?; + assert_eq!(resp.status(), 200, "adduser {g}/{email}"); + } + + let resp = authed(client().post(format!("{ws_base}/edit_instance_groups"))) + .json(&json!({ + "groups": ["top_admins", "base_devs"], + "roles": { "top_admins": "admin", "base_devs": "developer" } + })) + .send() + .await?; + assert_eq!(resp.status(), 200, "edit: {}", resp.text().await?); + + let (is_admin, via): (bool, Option) = sqlx::query_as( + "SELECT is_admin, added_via->>'group' FROM usr + WHERE workspace_id = 'test-workspace' AND email = 'demoted@example.com'", + ) + .fetch_one(&db) + .await?; + assert!(is_admin, "demoted@ should start as admin via top_admins"); + assert_eq!(via.as_deref(), Some("top_admins")); + + // Workspace state that must survive the demotion. + let username: String = sqlx::query_scalar( + "SELECT username FROM usr WHERE workspace_id = 'test-workspace' AND email = 'demoted@example.com'", + ) + .fetch_one(&db) + .await?; + sqlx::query( + "INSERT INTO favorite (workspace_id, usr, path, favorite_kind) + VALUES ('test-workspace', $1, 'f/lifecycle/keep', 'script')", + ) + .bind(&username) + .execute(&db) + .await?; + + // The import retains both groups but drops demoted@ from top_admins and dropped@ from + // base_devs. + let resp = authed(client().post(format!("{global_base}/overwrite"))) + .json(&json!([ + { "name": "top_admins", "emails": [] }, + { "name": "base_devs", "emails": ["demoted@example.com"] } + ])) + .send() + .await?; + assert_eq!(resp.status(), 200, "overwrite: {}", resp.text().await?); + + // demoted@ stays, re-roled to base_devs' developer, with their data intact. + let (is_admin, is_operator, via): (bool, bool, Option) = sqlx::query_as( + "SELECT is_admin, operator, added_via->>'group' FROM usr + WHERE workspace_id = 'test-workspace' AND email = 'demoted@example.com'", + ) + .fetch_one(&db) + .await?; + assert!( + !is_admin, + "admin from top_admins must not survive being dropped from it" + ); + assert!(!is_operator, "base_devs grants developer"); + assert_eq!(via.as_deref(), Some("base_devs")); + + let favorites: i64 = sqlx::query_scalar( + "SELECT count(*) FROM favorite WHERE workspace_id = 'test-workspace' AND path = 'f/lifecycle/keep'", + ) + .fetch_one(&db) + .await?; + assert_eq!( + favorites, 1, + "re-roling in place must not destroy workspace data" + ); + + // dropped@ lost their only configured group even though the group itself was retained. + let remaining: i64 = sqlx::query_scalar( + "SELECT count(*) FROM usr + WHERE workspace_id = 'test-workspace' AND email = 'dropped@example.com'", + ) + .fetch_one(&db) + .await?; + assert_eq!( + remaining, 0, + "member dropped from a retained group must be removed" + ); + + // Both groups were retained, so the workspace config is untouched. + let (groups, roles): (serde_json::Value, serde_json::Value) = sqlx::query_as( + "SELECT auto_invite->'instance_groups', auto_invite->'instance_groups_roles' + FROM workspace_settings WHERE workspace_id = 'test-workspace'", + ) + .fetch_one(&db) + .await?; + assert_eq!(groups, json!(["top_admins", "base_devs"])); + assert_eq!( + roles, + json!({ "top_admins": "admin", "base_devs": "developer" }) + ); + + Ok(()) +} + +/// Members whose `added_via` source is not 'instance_group' — manually added users, and the +/// orphaned members the `preserve_orphaned_instance_group_members` migration converted to +/// manual — are invisible to reconciliation: never re-roled and never removed, even when they +/// also appear in a configured group's membership. +#[cfg(feature = "private")] +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_reconcile_ignores_non_instance_group_members( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let global_base = format!("http://localhost:{port}/api/groups"); + let ws_base = format!("http://localhost:{port}/api/w/test-workspace/workspaces"); + + let resp = authed(client().post(format!("{global_base}/create"))) + .json(&json!({ "name": "visible_grp" })) + .send() + .await?; + assert_eq!(resp.status(), 200, "create"); + for email in ["kept@example.com", "shielded@example.com"] { + let resp = authed(client().post(format!("{global_base}/adduser/visible_grp"))) + .json(&json!({ "email": email })) + .send() + .await?; + assert_eq!(resp.status(), 200, "adduser {email}"); + } + + // shielded@ is already in the workspace through a non-instance_group source (the shape + // the migration leaves behind), at a role the group config would not grant. The username + // deliberately differs from the instance-derived one ('shielded'): an unguarded + // auto_add_user would then insert a second usr row for the email instead of no-op'ing on + // a username conflict, so the count assertions below can catch it. + sqlx::query( + r#"INSERT INTO usr (workspace_id, username, email, is_admin, operator, added_via) + VALUES ('test-workspace', 'shielded_legacy', 'shielded@example.com', true, false, + '{"source": "manual", "migrated_from_instance_group": "gone_grp"}'::jsonb)"#, + ) + .execute(&db) + .await?; + + let resp = authed(client().post(format!("{ws_base}/edit_instance_groups"))) + .json(&json!({ + "groups": ["visible_grp"], + "roles": { "visible_grp": "developer" } + })) + .send() + .await?; + assert_eq!(resp.status(), 200, "edit: {}", resp.text().await?); + + // kept@ was auto-added via the group; shielded@ kept their single manual row untouched. + let kept: i64 = sqlx::query_scalar( + "SELECT count(*) FROM usr WHERE workspace_id = 'test-workspace' AND email = 'kept@example.com' + AND added_via->>'source' = 'instance_group'", + ) + .fetch_one(&db) + .await?; + assert_eq!(kept, 1, "group member should be auto-added"); + + let shielded_rows: i64 = sqlx::query_scalar( + "SELECT count(*) FROM usr WHERE workspace_id = 'test-workspace' AND email = 'shielded@example.com'", + ) + .fetch_one(&db) + .await?; + assert_eq!( + shielded_rows, 1, + "reconciliation must not create a second usr row for a member already present under a non-instance_group source" + ); + + // Dropping both users from the group removes the instance_group-sourced member but must + // leave the manual row alone. + for email in ["kept@example.com", "shielded@example.com"] { + let resp = authed(client().post(format!("{global_base}/removeuser/visible_grp"))) + .json(&json!({ "email": email })) + .send() + .await?; + assert_eq!(resp.status(), 200, "removeuser {email}"); + } + + let kept: i64 = sqlx::query_scalar( + "SELECT count(*) FROM usr WHERE workspace_id = 'test-workspace' AND email = 'kept@example.com'", + ) + .fetch_one(&db) + .await?; + assert_eq!( + kept, 0, + "instance_group-sourced member loses access with their only group" + ); + + let (username, is_admin, via_source): (String, bool, Option) = sqlx::query_as( + "SELECT username, is_admin, added_via->>'source' FROM usr + WHERE workspace_id = 'test-workspace' AND email = 'shielded@example.com'", + ) + .fetch_one(&db) + .await?; + assert_eq!( + username, "shielded_legacy", + "the original manual row must be the only one" + ); + assert!( + is_admin, + "manual member's role must not be touched by reconciliation" + ); + assert_eq!(via_source.as_deref(), Some("manual")); + + Ok(()) +} + +/// The upgrade migration converts every member the reconciler would evict — those whose +/// granting group was deleted and those dropped from a group that still exists — and leaves +/// still-qualifying members alone. The migration has already run against the empty test +/// database by the time this executes, so the test fabricates pre-fix state and re-executes +/// the migration's statements, which are idempotent plain UPDATEs. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_preserve_orphaned_members_migration(db: Pool) -> anyhow::Result<()> { + // ghost_grp pins the statement order: it is referenced by the workspace and still has a + // membership row, but no instance_group row. Only when the reference strip runs before + // the conversion does ghost@ read as unconverted-by-membership nowhere and get preserved; + // converting first would spare them on the doomed reference and then strand them. + sqlx::raw_sql( + r#" + INSERT INTO workspace (id, name, owner) VALUES ('mig-ws', 'mig-ws', 'admin@windmill.dev'); + INSERT INTO workspace_settings (workspace_id, auto_invite) VALUES + ('mig-ws', '{"instance_groups": ["gone_grp", "ghost_grp", "live_grp"], "instance_groups_roles": {"gone_grp": "admin", "ghost_grp": "developer", "live_grp": "developer"}}'::jsonb); + INSERT INTO instance_group (name) VALUES ('live_grp'); + INSERT INTO email_to_igroup (email, igroup) VALUES + ('live@example.com', 'live_grp'), + ('ghost@example.com', 'ghost_grp'); + INSERT INTO usr (workspace_id, username, email, is_admin, operator, added_via) VALUES + ('mig-ws', 'orphan', 'orphan@example.com', true, false, '{"source": "instance_group", "group": "gone_grp"}'::jsonb), + ('mig-ws', 'droppedu', 'dropped@example.com', false, false, '{"source": "instance_group", "group": "live_grp"}'::jsonb), + ('mig-ws', 'ghostmember', 'ghost@example.com', false, false, '{"source": "instance_group", "group": "ghost_grp"}'::jsonb), + ('mig-ws', 'livemember', 'live@example.com', false, false, '{"source": "instance_group", "group": "live_grp"}'::jsonb); + "#, + ) + .execute(&db) + .await?; + + sqlx::raw_sql(include_str!( + "../../migrations/20260813195023_preserve_orphaned_instance_group_members.up.sql" + )) + .execute(&db) + .await?; + + // Deleted-group orphan and retained-group-dropped orphan both become manual members + // with the original group recorded; the still-qualifying member is untouched. + for (email, expected_group) in [ + ("orphan@example.com", "gone_grp"), + ("dropped@example.com", "live_grp"), + ("ghost@example.com", "ghost_grp"), + ] { + let (source, migrated_from): (Option, Option) = sqlx::query_as( + "SELECT added_via->>'source', added_via->>'migrated_from_instance_group' + FROM usr WHERE workspace_id = 'mig-ws' AND email = $1", + ) + .bind(email) + .fetch_one(&db) + .await?; + assert_eq!( + source.as_deref(), + Some("manual"), + "{email} should be converted" + ); + assert_eq!( + migrated_from.as_deref(), + Some(expected_group), + "{email} marker" + ); + } + + let (source, group): (Option, Option) = sqlx::query_as( + "SELECT added_via->>'source', added_via->>'group' + FROM usr WHERE workspace_id = 'mig-ws' AND email = 'live@example.com'", + ) + .fetch_one(&db) + .await?; + assert_eq!( + source.as_deref(), + Some("instance_group"), + "still-qualifying member spared" + ); + assert_eq!(group.as_deref(), Some("live_grp")); + + // The dangling references are stripped from both auto_invite fields; the live one stays. + let (groups, roles): (serde_json::Value, serde_json::Value) = sqlx::query_as( + "SELECT auto_invite->'instance_groups', auto_invite->'instance_groups_roles' + FROM workspace_settings WHERE workspace_id = 'mig-ws'", + ) + .fetch_one(&db) + .await?; + assert_eq!(groups, json!(["live_grp"])); + assert_eq!(roles, json!({ "live_grp": "developer" })); + + Ok(()) +} diff --git a/backend/windmill-api-schedule/src/lib.rs b/backend/windmill-api-schedule/src/lib.rs index b4e9bf269a..ec790ec060 100644 --- a/backend/windmill-api-schedule/src/lib.rs +++ b/backend/windmill-api-schedule/src/lib.rs @@ -27,6 +27,9 @@ use windmill_common::{ db::UserDB, error::{Error, JsonResult, Result}, schedule::Schedule, + trigger_history::{ + self, TriggerHistoryEvent, TriggerOperation, TriggerSource, SCHEDULE_TRIGGER_KIND, + }, user_drafts::{ delete_all_drafts_for_path, fetch_draft_only_list_rows, overlay_or_draft_only, UserDraftItemKind, WithDraftOverlay, WithDraftQuery, @@ -85,6 +88,40 @@ fn resolve_edited_by(authed: &ApiAuthed) -> String { authed.username.clone() } +/// Append this mutation to `trigger_history`, diffing the row against `before`. +/// +/// Call it on the transaction that made the change, after the change: the +/// snapshot it takes is the "after" side of the diff, and the two commit or roll +/// back together. +async fn record_schedule_history( + tx: &mut sqlx::PgConnection, + authed: &ApiAuthed, + w_id: &str, + path: &str, + operation: TriggerOperation, + before: Option, +) -> Result<()> { + let after = trigger_history::snapshot_row(&mut *tx, "schedule", w_id, path).await?; + // Nothing to describe when the row is not there after the write: the same + // guard the trigger side needs, kept here so the two read alike. + if after.is_none() { + return Ok(()); + } + trigger_history::record( + &mut *tx, + TriggerHistoryEvent { + workspace_id: w_id, + trigger_kind: SCHEDULE_TRIGGER_KIND, + path, + operation, + source: TriggerSource::of_request(authed.is_session_token), + username: Some(&authed.username), + changes: trigger_history::summarize_changes(before.as_ref(), after.as_ref()), + }, + ) + .await +} + pub fn workspaced_service() -> Router { Router::new() .route("/list", get(list_schedule)) @@ -417,6 +454,16 @@ async fn create_schedule( .await .map_err(|e| Error::internal_err(format!("inserting schedule in {w_id}: {e:#}")))?; + record_schedule_history( + &mut *tx, + &authed, + &w_id, + &ns.path, + TriggerOperation::Create, + None, + ) + .await?; + audit_log( &mut *tx, &authed, @@ -524,6 +571,8 @@ async fn edit_schedule( authed.email.clone() }; + let before = trigger_history::snapshot_row(&mut *tx, "schedule", &w_id, path).await?; + let schedule = sqlx::query_as!( Schedule, r#" @@ -632,6 +681,16 @@ async fn edit_schedule( // like set_enabled, flow updates, and worker job completions. clear_schedule(&mut tx, path, &w_id).await?; + record_schedule_history( + &mut *tx, + &authed, + &w_id, + path, + TriggerOperation::Update, + before, + ) + .await?; + audit_log( &mut *tx, &authed, @@ -1084,6 +1143,8 @@ pub async fn set_enabled( } } } + let before = trigger_history::snapshot_row(&mut *tx, "schedule", &w_id, path).await?; + // email is still written for backwards compat with old workers that don't know about permissioned_as let schedule_o = sqlx::query_as!( Schedule, @@ -1139,6 +1200,20 @@ pub async fn set_enabled( clear_schedule(&mut tx, path, &w_id).await?; + record_schedule_history( + &mut *tx, + &authed, + &w_id, + path, + if payload.enabled { + TriggerOperation::Enable + } else { + TriggerOperation::Disable + }, + before, + ) + .await?; + audit_log( &mut *tx, &authed, @@ -1285,6 +1360,22 @@ async fn delete_schedule( .await?; } + // No diff: the row is gone, and the trashbin above already keeps its full + // contents for a restore. + trigger_history::record( + &mut *tx, + TriggerHistoryEvent { + workspace_id: &w_id, + trigger_kind: SCHEDULE_TRIGGER_KIND, + path, + operation: TriggerOperation::Delete, + source: TriggerSource::of_request(authed.is_session_token), + username: Some(&authed.username), + changes: None, + }, + ) + .await?; + audit_log( &mut *tx, &authed, @@ -1373,6 +1464,11 @@ async fn set_default_error_handler( } if payload.override_existing { + // The rewrite and its history rows go in one transaction: on separate + // connections a concurrent edit could interleave, leaving the + // id-ordered drawer showing the wrong latest change, and a failed + // insert would leave the schedules rewritten with nothing recording it. + let mut tx = db.begin().await?; let updated_schedules: Vec; match payload.handler_type { HandlerType::Error => { @@ -1386,14 +1482,14 @@ async fn set_default_error_handler( payload.number_of_occurence_exact, w_id, ) - .fetch_all(&db) + .fetch_all(&mut *tx) .await?; } else { updated_schedules = sqlx::query_scalar!( "UPDATE schedule SET ws_error_handler_muted = false, on_failure = NULL, on_failure_extra_args = NULL, on_failure_times = NULL, on_failure_exact = NULL WHERE workspace_id = $1 RETURNING path", w_id, ) - .fetch_all(&db) + .fetch_all(&mut *tx) .await?; } } @@ -1406,14 +1502,14 @@ async fn set_default_error_handler( payload.number_of_occurence, w_id, ) - .fetch_all(&db) + .fetch_all(&mut *tx) .await?; } else { updated_schedules = sqlx::query_scalar!( "UPDATE schedule SET on_recovery = NULL, on_recovery_extra_args = NULL, on_recovery_times = NULL WHERE workspace_id = $1 RETURNING path", w_id, ) - .fetch_all(&db) + .fetch_all(&mut *tx) .await?; } } @@ -1425,18 +1521,70 @@ async fn set_default_error_handler( payload.extra_args, w_id, ) - .fetch_all(&db) + .fetch_all(&mut *tx) .await?; } else { updated_schedules = sqlx::query_scalar!( "UPDATE schedule SET on_success = NULL, on_success_extra_args = NULL WHERE workspace_id = $1 RETURNING path", w_id, ) - .fetch_all(&db) + .fetch_all(&mut *tx) .await?; } } } + // One row per schedule the workspace-wide override rewrote, so a handler + // that appeared on a schedule nobody edited is traceable. Every column + // the UPDATE above wrote, not just the handler path: the mute flag and + // the occurrence thresholds are what someone auditing a surprise + // notification change most needs. No `old` side and no + // already-had-this-value filter — the UPDATE rewrites the whole + // workspace unconditionally, so these rows record the write rather than + // a delta. + // Built from the same values the branch that ran actually bound: a reset + // (`payload.path` absent) hardcodes NULL / false in SQL while the request + // still carries the form's other fields, so reading them here would name + // values the write never produced. + let cleared = payload.path.is_none(); + let handler_path = payload.path.clone(); + let extra_args = (!cleared).then(|| payload.extra_args.clone()).flatten(); + let times = (!cleared).then_some(payload.number_of_occurence).flatten(); + let handler_fields = match payload.handler_type { + HandlerType::Error => serde_json::json!({ + "on_failure": { "new": handler_path }, + "on_failure_extra_args": { "new": extra_args }, + "on_failure_times": { "new": times }, + "on_failure_exact": { + "new": (!cleared).then_some(payload.number_of_occurence_exact).flatten() + }, + "ws_error_handler_muted": { + "new": !cleared && payload.workspace_handler_muted.unwrap_or(false) + }, + }), + HandlerType::Recovery => serde_json::json!({ + "on_recovery": { "new": handler_path }, + "on_recovery_extra_args": { "new": extra_args }, + "on_recovery_times": { "new": times }, + }), + HandlerType::Success => serde_json::json!({ + "on_success": { "new": handler_path }, + "on_success_extra_args": { "new": extra_args }, + }), + }; + trigger_history::record_bulk( + &mut tx, + &w_id, + SCHEDULE_TRIGGER_KIND, + &updated_schedules, + TriggerOperation::Update, + TriggerSource::of_request(authed.is_session_token), + Some(&authed.username), + Some(handler_fields), + ) + .await?; + + tx.commit().await?; + for updated_schedule_path in updated_schedules { // managed ducklake-maintenance rows get the handler update (their // failures should reach workspace handlers) but must not be diff --git a/backend/windmill-api-users/Cargo.toml b/backend/windmill-api-users/Cargo.toml index c322d6cfda..13ab8143d8 100644 --- a/backend/windmill-api-users/Cargo.toml +++ b/backend/windmill-api-users/Cargo.toml @@ -21,7 +21,6 @@ windmill-api-auth.workspace = true windmill-audit.workspace = true windmill-git-sync.workspace = true -dashmap.workspace = true argon2.workspace = true axum.workspace = true chrono.workspace = true diff --git a/backend/windmill-api-users/src/users.rs b/backend/windmill-api-users/src/users.rs index 96b89e322c..67e1721109 100644 --- a/backend/windmill-api-users/src/users.rs +++ b/backend/windmill-api-users/src/users.rs @@ -46,6 +46,7 @@ use windmill_common::audit::AuditAuthor; use windmill_common::auth::{safe_token_prefix, TOKEN_PREFIX_LEN}; use windmill_common::global_settings::AUTOMATE_USERNAME_CREATION_SETTING; use windmill_common::oauth2::InstanceEvent; +use windmill_common::per_minute_counter::PerMinuteCounter; use windmill_common::users::truncate_token; use windmill_common::users::COOKIE_NAME; use windmill_common::users::{ @@ -67,41 +68,24 @@ use windmill_git_sync::handle_deployment_metadata; pub const COOKIE_PATH: &str = "/"; -const TOKEN_CREATE_LIMIT_PER_MINUTE: i32 = 10; +const TOKEN_CREATE_LIMIT_PER_MINUTE: u32 = 10; -struct TokenRateLimitEntry { - count: i32, - minute_bucket: i64, -} - -static TOKEN_CREATE_RATE_LIMIT: LazyLock> = - LazyLock::new(dashmap::DashMap::new); +static TOKEN_CREATE_RATE_LIMIT: LazyLock> = + LazyLock::new(PerMinuteCounter::new); fn check_token_create_rate_limit(username: &str) -> Result<()> { if !*CLOUD_HOSTED { return Ok(()); } - let current_minute = chrono::Utc::now().timestamp() / 60; - - let mut entry = TOKEN_CREATE_RATE_LIMIT - .entry(username.to_string()) - .or_insert(TokenRateLimitEntry { count: 0, minute_bucket: current_minute }); - - if entry.minute_bucket != current_minute { - entry.count = 0; - entry.minute_bucket = current_minute; + if TOKEN_CREATE_RATE_LIMIT.try_increment(username.to_string(), TOKEN_CREATE_LIMIT_PER_MINUTE) { + return Ok(()); } - if entry.count >= TOKEN_CREATE_LIMIT_PER_MINUTE { - return Err(Error::Generic( - StatusCode::TOO_MANY_REQUESTS, - "Too many token creation requests. Please try again later.".to_string(), - )); - } - - entry.count += 1; - Ok(()) + Err(Error::Generic( + StatusCode::TOO_MANY_REQUESTS, + "Too many token creation requests. Please try again later.".to_string(), + )) } pub fn workspaced_service() -> Router { @@ -1386,7 +1370,7 @@ async fn convert_user_to_group( )); } - // Determine the group with highest precedence (same logic as process_instance_group_auto_adds) + // Determine the group with highest precedence (same logic as reconcile_workspace_instance_groups) let roles: std::collections::HashMap = if let Some(roles_json) = &eligible_groups[0].instance_groups_roles { serde_json::from_value(roles_json.clone()).unwrap_or_default() diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 71b74af834..6b84c8bdba 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -11615,41 +11615,6 @@ struct LogFeatureUsagePayload { events: Vec, } -// Only registered (feature, kind) actions are accepted, so telemetry stays -// limited to predefined feature actions. Keys are shape-checked (identifier-like, -// no spaces) rather than pinned to value sets: they come from our own frontend -// (modes, tab/draft kinds, tool names, provider:model) and pinning every value -// server-side was not worth the maintenance. -const FEATURE_USAGE_KINDS: &[(&str, &str)] = &[ - ("ai_session", "created"), - ("ai_session", "message"), - ("ai_session", "autonomy"), - ("ai_session", "tab"), - ("ai_session", "tokens"), - ("ai_session", "deployed"), - ("ai_session", "archived"), - ("ai_session", "deleted"), - ("ai_session", "beta_optout"), - ("ai_session", "beta_optin"), - ("ai_chat", "message"), - ("ai_chat", "model"), - ("ai_chat", "tool"), - ("flow_editor", "panel_placement"), -]; - -fn is_identifier_shaped(s: &str, max_len: usize) -> bool { - !s.is_empty() - && s.len() <= max_len - && s.chars() - .all(|c| c.is_ascii_alphanumeric() || matches!(c, '_' | '-' | ':' | '.' | '/')) -} - -fn valid_feature_usage_event(e: &FeatureUsageEvent) -> bool { - FEATURE_USAGE_KINDS.contains(&(e.feature.as_str(), e.kind.as_str())) - && (e.key.is_empty() || is_identifier_shaped(&e.key, 100)) - && (e.entity_id.is_empty() || is_identifier_shaped(&e.entity_id, 50)) -} - async fn log_feature_usage( Extension(db): Extension, Json(payload): Json, @@ -11658,7 +11623,15 @@ async fn log_feature_usage( // single INSERT error out ("cannot affect row a second time"). let mut agg: HashMap<(String, String, String, String), i64> = HashMap::new(); for e in payload.events.into_iter().take(MAX_FEATURE_USAGE_EVENTS) { - if !valid_feature_usage_event(&e) { + // Which actions may be recorded lives in + // `windmill_common::feature_usage`, shared with the in-process writer so + // both admit exactly the same events. + if !windmill_common::feature_usage::is_recordable_event( + &e.feature, + &e.kind, + &e.key, + &e.entity_id, + ) { continue; } let value = e.value.unwrap_or(1).clamp(1, 1_000_000); @@ -11668,12 +11641,18 @@ async fn log_feature_usage( if agg.is_empty() { return Ok(StatusCode::NO_CONTENT); } - let mut features = Vec::with_capacity(agg.len()); - let mut kinds = Vec::with_capacity(agg.len()); - let mut keys = Vec::with_capacity(agg.len()); - let mut entity_ids = Vec::with_capacity(agg.len()); - let mut values = Vec::with_capacity(agg.len()); - for ((feature, kind, key, entity_id), value) in agg { + // Sorted for the same reason as `flush_feature_usage`: this endpoint and the + // backend flusher upsert the same rows, and two batches touching them in + // opposite orders deadlock. + let mut rows: Vec<((String, String, String, String), i64)> = agg.into_iter().collect(); + rows.sort_unstable_by(|a, b| a.0.cmp(&b.0)); + + let mut features = Vec::with_capacity(rows.len()); + let mut kinds = Vec::with_capacity(rows.len()); + let mut keys = Vec::with_capacity(rows.len()); + let mut entity_ids = Vec::with_capacity(rows.len()); + let mut values = Vec::with_capacity(rows.len()); + for ((feature, kind, key, entity_id), value) in rows { features.push(feature); kinds.push(kind); keys.push(key); diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 46720669d3..0db6e99cf0 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -268,6 +268,15 @@ paths: type: string - $ref: "#/components/parameters/ResourceName" - $ref: "#/components/parameters/ActionKind" + - name: before_id + in: query + description: > + only return logs with an id strictly lower than this one. Logs are ordered by + descending id, so this is a keyset cursor to stream a page in several batches + without paying a growing offset. + schema: + type: integer + format: int64 - name: all_workspaces in: query description: get audit logs for all workspaces @@ -8083,11 +8092,72 @@ paths: type: string description: type: string - parameters: + inputSchema: type: object + annotations: + type: object + properties: + title: + type: string + readOnlyHint: + type: boolean + destructiveHint: + type: boolean + idempotentHint: + type: boolean + openWorldHint: + type: boolean required: - name - - parameters + - inputSchema + + /w/{workspace}/resources/mcp_call_tool/{path}: + post: + summary: call a tool on the MCP server described by the resource + operationId: callMcpTool + tags: + - resource + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Path" + requestBody: + description: tool name and arguments + required: true + content: + application/json: + schema: + type: object + properties: + tool: + type: string + arguments: + type: object + read_only: + type: boolean + description: | + set when the caller ran the tool without asking the user to + confirm it; the call is refused unless the server's live + listing marks the tool read-only + required: + - tool + responses: + "200": + description: | + the MCP tool result, forwarded verbatim. A tool that ran but failed + returns 200 with isError true. + content: + application/json: + schema: + type: object + properties: + content: + type: array + items: + type: object + structuredContent: + type: object + isError: + type: boolean /w/{workspace}/resources/list_names/{name}: get: @@ -20353,6 +20423,36 @@ paths: type: string nullable: true + /w/{workspace}/triggers_history/list: + get: + summary: list the history of schedule and trigger modifications + operationId: listTriggerHistory + tags: + - trigger + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Page" + - $ref: "#/components/parameters/PerPage" + - name: trigger_kind + description: "'schedule' or a trigger type (http, kafka, ...)" + in: query + schema: + type: string + - name: path + description: only return the history of the trigger at this path + in: query + schema: + type: string + responses: + "200": + description: trigger history + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/TriggerHistoryEntry" + /w/{workspace}/folders/list: get: summary: list folders @@ -28256,6 +28356,44 @@ components: is_fileset: type: boolean + TriggerHistoryEntry: + type: object + properties: + id: + type: integer + format: int64 + trigger_kind: + type: string + description: "'schedule' or a trigger type (http, kafka, ...)" + path: + type: string + operation: + type: string + enum: [create, update, delete, enable, disable, suspend] + source: + type: string + description: The kind of client the change came from. `worker` means the server disabled the trigger on its own after a failure. + enum: [ui, cli, api, worker] + username: + type: string + nullable: true + description: Unset when the server acted on its own. + created_at: + type: string + format: date-time + changes: + type: object + nullable: true + additionalProperties: true + description: "{field: {old, new}} for the fields that actually changed. Unset for a delete." + required: + - id + - trigger_kind + - path + - operation + - source + - created_at + Schedule: type: object properties: diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index acfdc6639b..b79e0b8b14 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -78,7 +78,7 @@ use crate::{ users::{ get_scope_tags, require_owner_of_path, require_path_read_access_for_preview, OptAuthed, }, - utils::{check_scopes, content_plain, require_super_admin}, + utils::{build_scope_path_predicate, check_scopes, content_plain, require_super_admin}, }; use anyhow::Context; use axum::{ @@ -1458,6 +1458,11 @@ pub(crate) async fn require_job_read_access( } } + // A path-scoped `jobs:run` token is likewise hard-restricted to the runnables it + // may start, ahead of every grant below — the token is handed out to run one thing, + // so it must not read jobs of anything else merely because its owner could. + require_job_within_run_scope(db, authed, w_id, job_id).await?; + // Fast path: you can always read a job you launched. This is also load-bearing // for apps — a component job runs as the app policy's `permissioned_as`, but its // `created_by` is the launching viewer, so the RLS probe below would hide it. @@ -1583,6 +1588,94 @@ pub(crate) async fn require_job_read_access( } } +/// Confines a path-scoped `jobs:run::` token to jobs of the runnables it may +/// start. Such a token is minted per script/flow for a webhook or CI caller, which needs +/// to start that runnable and poll the resulting job — nothing more. Without this, the +/// by-id read routes it reaches for polling (`RUN_WHITELISTED_GET_PATHS`) would serve +/// it the args/result/logs of any job its owner's identity can see, defeating the path +/// confinement the scope exists to provide. +/// +/// The scope may be satisfied by the job itself or by any of its `parent_job` ancestors: +/// a flow step's `runnable_path` is the inner runnable's, so a `jobs:run:flows:` +/// token inspecting its own run must still reach the steps beneath it. +/// +/// An `apps:run|write:` scope is a start grant too, so a job an app launched +/// (`trigger_kind = 'app'`, an app-provenance stamp `/jobs/run` cannot forge) satisfies +/// the confinement for a token scoped to that app. Without this, a token holding both +/// could start an app's inline-script component but not read the run back — those jobs +/// are `AppScript`/`Preview` kinds that no `jobs:run` scope can name. +/// +/// No-op — and no query — for every caller whose job reads are not run-confined (see +/// `job_read_run_confinement`), which is all sessions, unscoped tokens and `jobs:read` +/// tokens. +async fn require_job_within_run_scope( + db: &DB, + authed: &ApiAuthed, + w_id: &str, + job_id: &Uuid, +) -> error::Result<()> { + let Some(confinement) = + windmill_api_auth::scopes::job_read_run_confinement(authed.scopes.as_deref()) + else { + return Ok(()); + }; + // `scope_kind` is the runnable kind a `jobs:run::` scope can name, or + // NULL for a job no such scope reaches directly (previews, dependency jobs, + // flow-inlined scripts) — those are still readable as a step of a matching flow, + // through their ancestors. A `singlestepflow` wraps either a script or a flow, so it + // projects onto the wrapped runnable the same way the batch-rerun query does. + let chain = sqlx::query!( + r#"WITH RECURSIVE chain(id, parent_job) AS ( + SELECT id, parent_job FROM v2_job WHERE id = $1 AND workspace_id = $2 + UNION ALL + SELECT j.id, j.parent_job FROM v2_job j + JOIN chain c ON j.id = c.parent_job AND j.workspace_id = $2 + ) + SELECT j.runnable_path, + CASE + WHEN j.kind IN ('script', 'script_hub', 'unassigned_script') THEN 'scripts' + WHEN j.kind IN ('flow', 'unassigned_flow') THEN 'flows' + WHEN j.kind IN ('singlestepflow', 'unassigned_singlestepflow') THEN + CASE WHEN COALESCE( + (SELECT m->'value'->>'type' + FROM jsonb_array_elements(j.raw_flow->'modules') m + WHERE m->>'id' IN ('a', 'main') + LIMIT 1), + 'script' + ) = 'flow' THEN 'flows' ELSE 'scripts' END + END AS scope_kind, + CASE WHEN j.trigger_kind = 'app' THEN j.trigger END AS launched_by_app + FROM v2_job j JOIN chain c ON c.id = j.id + WHERE j.workspace_id = $2"#, + job_id, + w_id, + ) + .fetch_all(db) + .await?; + + let runs_app = build_scope_path_predicate(authed, "apps", "run"); + let in_scope = chain.iter().any(|job| { + match (job.runnable_path.as_deref(), job.scope_kind.as_deref()) { + (Some(runnable_path), Some(kind)) + if windmill_api_auth::scopes::run_confinement_admits( + &confinement, + kind, + runnable_path, + ) => + { + true + } + _ => job.launched_by_app.as_deref().is_some_and(&runs_app), + } + }); + + if in_scope { + Ok(()) + } else { + Err(Error::NotFound(format!("Job {job_id} not found"))) + } +} + /// Self + every `parent_job` ancestor (intermediate sub-flows up to the top-level /// root) of `job_id`, resolved via the root DB (flow lineage is not sensitive). /// Falls back to `[job_id]` if the row is absent so callers still run their probe. @@ -1902,7 +1995,15 @@ async fn get_job( // same visibility as `jobs/list` (see `require_job_read_access`), or hold a public // share link when logged out — which `public_view_grant` already established above, // so skip re-deriving it here: this handler is what the public run page polls. - if !has_valid_approval_token && !public_view_grant { + if has_valid_approval_token || public_view_grant { + // Both grants skip the gate below, and with it the run-scope confinement that + // gate carries. That confinement is a hard restriction, so re-apply it: holding + // an approval link for a job must not let a scoped token read one outside the + // runnables it may start. + if let Some(authed) = opt_authed.as_ref() { + require_job_within_run_scope(&db, authed, &w_id, &id).await?; + } + } else { require_opt_authed_job_read_access( &db, &user_db, @@ -5276,6 +5377,15 @@ pub async fn get_suspended_job_flow( .flatten() .ok_or_else(|| anyhow::anyhow!("parent flow job not found"))?; + // The resume secret is this route's gate, so it never reaches + // `require_job_read_access` and the run-scope confinement that gate carries. Re-apply + // it against the flow whose args and status are about to be returned: holding a + // resume secret must not let a scoped token read a flow it may not run. Anonymous + // approvers are unaffected. + if let Some(authed) = authed.as_ref() { + require_job_within_run_scope(&db, authed, &w_id, &flow_id).await?; + } + let flow = GetQuery::new() .without_logs() .without_code() @@ -5455,9 +5565,14 @@ pub async fn create_job_signature( pub async fn get_flow_user_state( authed: ApiAuthed, + Extension(db): Extension, Extension(user_db): Extension, Path((w_id, job_id, key)): Path<(String, Uuid, String)>, ) -> error::JsonResult> { + // Reachable by a `jobs:run` token (it is one of the by-id routes a run needs), so + // apply the same run-scope confinement as the other single-job reads. RLS below + // still governs which jobs the owner's identity can see at all. + require_job_within_run_scope(&db, &authed, &w_id, &job_id).await?; let mut tx = user_db.begin(&authed).await?; let r = sqlx::query_scalar!( r#" @@ -10597,7 +10712,14 @@ async fn get_completed_job_result( _ => false, }; - if !approval_secret_ok { + if approval_secret_ok { + // The approval secret skips the gate below, and with it the run-scope + // confinement that gate carries — re-apply it, as `get_job` does for the + // approval token. Anonymous approval access is untouched. + if let Some(authed) = opt_authed.as_ref() { + require_job_within_run_scope(&db, authed, &w_id, &id).await?; + } + } else { require_opt_authed_job_read_access( &db, &user_db, diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index e67420c53c..44d1d7cb2b 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -178,6 +178,7 @@ mod teams_oss; mod token; mod tracing_init; mod trash; +mod trigger_history; pub mod triggers; mod users; #[cfg(feature = "private")] @@ -280,6 +281,23 @@ async fn set_deploy_origin( windmill_common::deploy_origin::scope(origin, next.run(req)).await } +/// Scope the request in the client kind it declares, so a trigger mutation can +/// be attributed to the CLI rather than to a bare API call. Entered for every +/// request, undeclared ones included: `TriggerSource::of_request` reads the +/// scope's absence as "no request is being served", which is what separates a +/// caller from a worker disabling a trigger on its own. +async fn set_request_client( + req: axum::extract::Request, + next: axum::middleware::Next, +) -> axum::response::Response { + let client = req + .headers() + .get(windmill_common::trigger_history::CLIENT_HEADER) + .and_then(|v| v.to_str().ok()) + .and_then(windmill_common::trigger_history::client_from_header); + windmill_common::trigger_history::scope_client(client, next.run(req)).await +} + #[cfg(not(feature = "tantivy"))] type IndexReader = (); @@ -639,6 +657,7 @@ pub async fn run_server( .nest("/folders_history", folder_history::workspaced_service()) .nest("/groups", groups::workspaced_service()) .nest("/groups_history", group_history::workspaced_service()) + .nest("/triggers_history", trigger_history::workspaced_service()) .nest("/inputs", windmill_api_inputs::workspaced_service()) .nest("/internal_db", internal_db::workspaced_service()) .route("/labels/list", get(list_workspace_labels)) @@ -1136,6 +1155,8 @@ pub async fn run_server( let app = app.layer(axum::middleware::from_fn(set_deploy_origin)); + let app = app.layer(axum::middleware::from_fn(set_request_client)); + let app = app.layer(CatchPanicLayer::custom(|err| { tracing::error!("panic in handler, returning 500: {:?}", err); Response::builder() @@ -1166,8 +1187,10 @@ pub async fn run_server( } // Announce this server is ready so coordinated restarts can detect a healthy peer. - if let Err(e) = announce_server_started(&db).await { - tracing::warn!("Failed to announce server started: {e:#}"); + if server_mode { + if let Err(e) = announce_server_started(&db).await { + tracing::warn!("Failed to announce server started: {e:#}"); + } } let server = server.with_graceful_shutdown(async move { @@ -1314,25 +1337,35 @@ pub async fn wait_for_db_migrations( const SERVER_HEARTBEAT_TASK: &str = "server_heartbeat"; -/// Write a server-started heartbeat to `background_task_state` so that -/// other instances waiting to restart can detect this server is healthy. +/// Write a server-started heartbeat to `background_task_state` so that other +/// traffic-serving instances waiting to restart can detect this one is healthy. +/// +/// Only `server_mode` processes announce, since only they are peers worth waiting for: +/// `spawn_graceful_killpill` holds a shutdown open to keep the API answered, and a worker, +/// indexer or MCP process coming up is no evidence that it is. +/// +/// The row is keyed per host and `owner` per process, and both halves carry weight. +/// `INSTANCE_NAME` is random per start, so a row keyed on it never conflicts and +/// accumulates one row per start; `owner` is what tells a peer's start from its own +/// when processes share a host. async fn announce_server_started(db: &DB) -> anyhow::Result<()> { - use windmill_common::INSTANCE_NAME; + use windmill_common::{utils::HOSTNAME, INSTANCE_NAME}; let instance = INSTANCE_NAME.as_str(); + let host = HOSTNAME.as_str(); sqlx::query( "INSERT INTO background_task_state (name, value, running, owner, started_at, updated_at) VALUES ($1, '\"started\"'::jsonb, true, $2, NOW(), NOW()) ON CONFLICT (name) - DO UPDATE SET updated_at = NOW(), running = true, owner = $2", + DO UPDATE SET started_at = NOW(), updated_at = NOW(), running = true, owner = $2", ) - .bind(format!("{SERVER_HEARTBEAT_TASK}:{instance}")) + .bind(format!("{SERVER_HEARTBEAT_TASK}:{host}")) .bind(instance) .execute(db) .await?; - tracing::info!("Announced server started for instance {instance}"); + tracing::info!("Announced server started for instance {instance} on host {host}"); Ok(()) } @@ -1362,10 +1395,10 @@ pub async fn check_any_server_started(db: &DB, not_before: chrono::DateTime not_before` (the moment a restart was initiated), diff --git a/backend/windmill-api/src/mcp_tools.rs b/backend/windmill-api/src/mcp_tools.rs index bba945ac54..e1931873a9 100644 --- a/backend/windmill-api/src/mcp_tools.rs +++ b/backend/windmill-api/src/mcp_tools.rs @@ -2,6 +2,7 @@ use axum::{ extract::{Extension, Path}, Json, }; +use serde::Deserialize; use serde_json::value::RawValue; use windmill_api_auth::{check_scopes, ApiAuthed}; use windmill_common::{ @@ -11,21 +12,46 @@ use windmill_common::{ }; use windmill_store::{resources::explain_resource_perm_error, variables::get_value_internal}; -pub(crate) async fn get_mcp_tools( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, -) -> JsonResult> { - let path = path.to_path(); - check_scopes(&authed, || format!("resources:read:{}", path))?; +/// A connected MCP server is a third party the user chose, reached over a +/// connection this request holds open: without a deadline one that never answers +/// pins an API worker and the chat turn behind it for as long as it likes. +const MCP_DEADLINE: std::time::Duration = std::time::Duration::from_secs(60); +/// Best-effort courtesy to the server, so it cannot extend the deadline above. +const MCP_SHUTDOWN_DEADLINE: std::time::Duration = std::time::Duration::from_secs(5); - let mut tx = user_db.clone().begin(&authed).await?; +async fn with_deadline( + what: &str, + fut: impl std::future::Future>, +) -> Result { + tokio::time::timeout(MCP_DEADLINE, fut) + .await + .map_err(|_| { + Error::ExecutionErr(format!( + "MCP server did not answer within {}s ({what})", + MCP_DEADLINE.as_secs() + )) + })? +} + +/// Connect to the MCP server described by the `mcp` resource at `path`. +/// +/// The caller is responsible for the scope check; everything else (resource +/// visibility, token resolution) goes through the caller's permissioned path so +/// the endpoint can never act as a confused deputy for a resource or secret the +/// caller cannot read. +async fn connect_mcp_client( + authed: &ApiAuthed, + db: &DB, + user_db: &UserDB, + w_id: &str, + path: &str, +) -> Result { + let mut tx = user_db.clone().begin(authed).await?; let resource_value_o = sqlx::query_scalar!( "SELECT value as \"value: sqlx::types::Json>\" FROM resource WHERE path = $1 AND workspace_id = $2", - &path, - &w_id + path, + w_id ) .fetch_optional(&mut *tx) .await?; @@ -33,7 +59,7 @@ pub(crate) async fn get_mcp_tools( tx.commit().await?; if resource_value_o.is_none() { - explain_resource_perm_error(&path, &w_id, &db, &authed).await?; + explain_resource_perm_error(path, w_id, db, authed).await?; } let resource_value = not_found_if_none(resource_value_o, "Resource", path)? @@ -58,20 +84,20 @@ pub(crate) async fn get_mcp_tools( WHERE variable.path = $1 AND variable.workspace_id = $2 "#, token_var_path, - &w_id + w_id ) - .fetch_optional(&db) + .fetch_optional(db) .await?; if let Some(info) = token_info { if let (Some(account_id), Some(true)) = (info.account_id, info.is_expired) { - let refresh_tx = user_db.clone().begin(&authed).await?; + let refresh_tx = user_db.clone().begin(authed).await?; if let Err(e) = crate::oauth2_oss::_refresh_token( refresh_tx, token_var_path, - &w_id, + w_id, account_id, - &db, + db, ) .await { @@ -93,17 +119,40 @@ pub(crate) async fn get_mcp_tools( if token_var_path.trim().is_empty() { None } else { - let db_authed = - DbWithOptAuthed::from_authed(&authed, db.clone(), Some(user_db.clone())); - Some(get_value_internal(&db_authed, &w_id, token_var_path, false).await?) + let db_authed = DbWithOptAuthed::from_authed(authed, db.clone(), Some(user_db.clone())); + Some(get_value_internal(&db_authed, w_id, token_var_path, false).await?) } } else { None }; - let client = windmill_mcp::McpClient::from_resource(mcp_resource, token) + windmill_mcp::McpClient::from_resource(mcp_resource, token) .await - .map_err(|e| Error::ExecutionErr(format!("Failed to connect to MCP server: {}", e)))?; + .map_err(|e| Error::ExecutionErr(format!("Failed to connect to MCP server: {}", e))) +} + +async fn shutdown_mcp_client(client: windmill_mcp::McpClient) { + match tokio::time::timeout(MCP_SHUTDOWN_DEADLINE, client.shutdown()).await { + Ok(Err(e)) => tracing::warn!("Failed to shutdown MCP client: {}", e), + Err(_) => tracing::warn!("MCP client shutdown timed out"), + Ok(Ok(())) => {} + } +} + +pub(crate) async fn get_mcp_tools( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, +) -> JsonResult> { + let path = path.to_path(); + check_scopes(&authed, || format!("resources:read:{}", path))?; + + let client = with_deadline( + "listing tools", + connect_mcp_client(&authed, &db, &user_db, &w_id, path), + ) + .await?; let tools: Vec = client .available_tools() @@ -114,9 +163,71 @@ pub(crate) async fn get_mcp_tools( }) .collect::>>()?; - if let Err(e) = client.shutdown().await { - tracing::warn!("Failed to shutdown MCP client: {}", e); - } + shutdown_mcp_client(client).await; Ok(Json(tools)) } + +#[derive(Deserialize)] +pub(crate) struct CallMcpToolRequest { + tool: String, + arguments: Option>, + /// Set by a caller that skipped the user's confirmation because it had + /// listed the tool as read-only. Verified below against the live listing. + read_only: Option, +} + +/// `readOnlyHint` is the server's own claim, so this cannot tell a hostile +/// server from an honest one; what it guarantees is that the claim comes from +/// the server about to be called, not from a listing of whatever the resource +/// pointed at when the caller cached it. +fn tool_is_read_only(client: &windmill_mcp::McpClient, tool: &str) -> bool { + client + .available_tools() + .iter() + .find(|t| t.name.as_ref() == tool) + .and_then(|t| t.annotations.as_ref()) + .and_then(|a| a.read_only_hint) + .unwrap_or(false) +} + +pub(crate) async fn call_mcp_tool( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, + Json(req): Json, +) -> JsonResult { + let path = path.to_path(); + check_scopes(&authed, || format!("resources:write:{}", path))?; + + let arguments = req.arguments.as_ref().map(|a| a.get()).unwrap_or("{}"); + // One deadline over the whole exchange (connect, then call), so a server that + // stalls after answering the handshake is bounded too. + let (client, result) = with_deadline(&format!("calling {}", req.tool), async { + let client = connect_mcp_client(&authed, &db, &user_db, &w_id, path).await?; + if req.read_only == Some(true) && !tool_is_read_only(&client, &req.tool) { + return Ok((client, None)); + } + let result = client.call_tool(&req.tool, arguments).await; + Ok((client, Some(result))) + }) + .await?; + + shutdown_mcp_client(client).await; + + let Some(result) = result else { + return Err(Error::BadRequest(format!( + "MCP tool {} is not marked read-only by the server, it must be called as a tool that modifies data", + req.tool + ))); + }; + + // A tool that ran but reported failure comes back as `Ok` with `isError: + // true` in the payload; forwarding it verbatim lets the caller show the + // server's own error text instead of a generic 500. + let result = result + .map_err(|e| Error::ExecutionErr(format!("Failed to call MCP tool {}: {}", req.tool, e)))?; + + Ok(Json(result)) +} diff --git a/backend/windmill-api/src/public_app_rate_limit.rs b/backend/windmill-api/src/public_app_rate_limit.rs index 5d49c17a71..baeb90ec0b 100644 --- a/backend/windmill-api/src/public_app_rate_limit.rs +++ b/backend/windmill-api/src/public_app_rate_limit.rs @@ -6,42 +6,27 @@ * LICENSE-AGPL for a copy of the license. */ -use chrono::Utc; -use dashmap::DashMap; use hyper::StatusCode; use std::sync::LazyLock; use windmill_common::error::{Error, Result}; +use windmill_common::per_minute_counter::PerMinuteCounter; -struct RateLimitEntry { - count: i32, - minute_bucket: i64, -} - -static RATE_LIMIT_COUNTER: LazyLock> = LazyLock::new(DashMap::new); +static RATE_LIMIT_COUNTER: LazyLock> = + LazyLock::new(PerMinuteCounter::new); pub fn check_and_increment(workspace_id: &str, limit: i32) -> Result<()> { - let current_minute = Utc::now().timestamp() / 60; - - let mut entry = RATE_LIMIT_COUNTER - .entry(workspace_id.to_string()) - .or_insert(RateLimitEntry { count: 0, minute_bucket: current_minute }); - - if entry.minute_bucket != current_minute { - entry.count = 0; - entry.minute_bucket = current_minute; + // Clamp before the cast: `as u32` on a negative limit wraps into an effectively unlimited + // allowance, where a non-positive limit must reject every execution. + if RATE_LIMIT_COUNTER.try_increment(workspace_id.to_string(), limit.max(0) as u32) { + return Ok(()); } - if entry.count >= limit { - return Err(Error::Generic( - StatusCode::TOO_MANY_REQUESTS, - format!( - "Rate limit exceeded for public app executions in workspace '{}'. \ - Limit: {} per minute per server.", - workspace_id, limit - ), - )); - } - - entry.count += 1; - Ok(()) + Err(Error::Generic( + StatusCode::TOO_MANY_REQUESTS, + format!( + "Rate limit exceeded for public app executions in workspace '{}'. \ + Limit: {} per minute per server.", + workspace_id, limit + ), + )) } diff --git a/backend/windmill-api/src/resources.rs b/backend/windmill-api/src/resources.rs index ee26ce758e..803927e43c 100644 --- a/backend/windmill-api/src/resources.rs +++ b/backend/windmill-api/src/resources.rs @@ -1,16 +1,18 @@ #[cfg(feature = "mcp")] -use axum::routing::get; +use axum::routing::{get, post}; use axum::Router; -/// Wraps the subcrate's workspaced_service with the mcp_tools route -/// that depends on windmill-api internals. +/// Wraps the subcrate's workspaced_service with the mcp_tools routes +/// that depend on windmill-api internals. pub fn workspaced_service() -> Router { let router = windmill_store::resources::workspaced_service(); #[cfg(feature = "mcp")] - use crate::mcp_tools::get_mcp_tools; + use crate::mcp_tools::{call_mcp_tool, get_mcp_tools}; #[cfg(feature = "mcp")] - let router = router.route("/mcp_tools/{*path}", get(get_mcp_tools)); + let router = router + .route("/mcp_tools/{*path}", get(get_mcp_tools)) + .route("/mcp_call_tool/{*path}", post(call_mcp_tool)); router } diff --git a/backend/windmill-api/src/token.rs b/backend/windmill-api/src/token.rs index acb65d6cf0..fc808767ec 100644 --- a/backend/windmill-api/src/token.rs +++ b/backend/windmill-api/src/token.rs @@ -236,6 +236,26 @@ lazy_static! { }], }); + // Read-only: `trigger_history` is append-only and written by the server + // alone, so there is no `triggers_history:write`. Its own domain rather + // than a `schedules`/`*_triggers` alias: one listing spans every kind, + // and a history row quotes the whole trigger row (a schedule's `args` + // included), so reading it is an explicit grant rather than a side + // effect of being able to read the trigger. Path-selectable because the + // route filters rows by the caller's path grants. + groups.push(ScopeDomain { + name: "Trigger History".to_string(), + description: Some( + "Read-only access to the modification history of schedules and triggers" + .to_string(), + ), + scopes: vec![ScopeOption { + value: "triggers_history:read".to_string(), + label: "Read".to_string(), + requires_resource_path: true, + }], + }); + groups.extend(build_standard_scope_domains()); groups.extend(build_trigger_scope_domains()); diff --git a/backend/windmill-api/src/trash.rs b/backend/windmill-api/src/trash.rs index 416546b850..53fc088582 100644 --- a/backend/windmill-api/src/trash.rs +++ b/backend/windmill-api/src/trash.rs @@ -9,6 +9,9 @@ use windmill_common::{ db::UserDB, error::{Error, Result}, trashbin::{self, TrashItem, TrashItemWithData}, + trigger_history::{ + self, TriggerHistoryEvent, TriggerOperation, TriggerSource, SCHEDULE_TRIGGER_KIND, + }, utils::require_admin, }; @@ -88,6 +91,31 @@ async fn restore_trash_item( .execute(&mut *tx) .await?; + // A restore puts the trigger back, so the history has to say so: otherwise + // the last thing it records for a live trigger is its own deletion. The + // trashed row is the snapshot, so this needs no extra read. + let restored_trigger_kind = match item.item_kind.as_str() { + SCHEDULE_TRIGGER_KIND => Some(SCHEDULE_TRIGGER_KIND), + // `_trigger` is what `delete_trigger` trashes it under, and the + // stem is the `TRIGGER_TYPE` the history records against. + kind => kind.strip_suffix("_trigger"), + }; + if let Some(trigger_kind) = restored_trigger_kind { + trigger_history::record( + &mut *tx, + TriggerHistoryEvent { + workspace_id: &w_id, + trigger_kind, + path: &item.item_path, + operation: TriggerOperation::Create, + source: TriggerSource::of_request(authed.is_session_token), + username: Some(&authed.username), + changes: trigger_history::summarize_changes(None, item.item_data.get("row")), + }, + ) + .await?; + } + audit_log( &mut *tx, &authed, diff --git a/backend/windmill-api/src/trigger_history.rs b/backend/windmill-api/src/trigger_history.rs new file mode 100644 index 0000000000..75ca03b90f --- /dev/null +++ b/backend/windmill-api/src/trigger_history.rs @@ -0,0 +1,105 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use axum::{ + extract::{Extension, Path, Query}, + routing::get, + Router, +}; +use serde::{Deserialize, Serialize}; +use windmill_api_auth::{build_scope_path_filter, check_scopes, ApiAuthed, ScopePathFilter}; +use windmill_common::{ + db::UserDB, + error::JsonResult, + utils::{paginate, Pagination}, +}; + +pub fn workspaced_service() -> Router { + Router::new().route("/list", get(list_trigger_history)) +} + +#[derive(Serialize)] +pub struct TriggerHistoryEntry { + pub id: i64, + pub trigger_kind: String, + pub path: String, + pub operation: String, + pub source: String, + pub username: Option, + pub created_at: chrono::DateTime, + pub changes: Option, +} + +#[derive(Deserialize)] +pub struct ListTriggerHistoryQuery { + pub page: Option, + pub per_page: Option, + /// `"schedule"` or a trigger type (`"http"`, `"kafka"`, …). + pub trigger_kind: Option, + pub path: Option, +} + +/// Two gates, because they answer different questions: the RLS policies on +/// `trigger_history` bound the rows to what the *user* may read, and +/// `triggers_history:read:` bounds them further to what this *token* may +/// read. Without the second, a token scoped to one path could read the diffs of +/// every trigger its user can see, and a `create` row quotes the whole trigger +/// row, a schedule's `args` included. +async fn list_trigger_history( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, + Query(query): Query, +) -> JsonResult> { + if let Some(path) = query.path.as_deref() { + check_scopes(&authed, || format!("triggers_history:read:{}", path))?; + } + + // In the WHERE, not a retain after the fetch: the result is paginated, and a + // post-fetch filter would let a page's size report how many rows the token + // may not read — and return short pages that read as "no history". + let (scope_all, scope_exact, scope_prefix) = + match build_scope_path_filter(&authed, "triggers_history", "read") { + ScopePathFilter::AllowAll => (true, Vec::new(), Vec::new()), + ScopePathFilter::Restricted { exact, prefix } => (false, exact, prefix), + }; + + let mut tx = user_db.begin(&authed).await?; + + let (per_page, offset) = paginate(Pagination { page: query.page, per_page: query.per_page }); + + let history = sqlx::query_as!( + TriggerHistoryEntry, + "SELECT id, trigger_kind, path, operation, source, username, created_at, changes + FROM trigger_history + WHERE workspace_id = $1 + AND ($2::TEXT IS NULL OR trigger_kind = $2) + AND ($3::TEXT IS NULL OR path = $3) + AND ( $6 + OR path = ANY($7) + OR EXISTS ( SELECT 1 FROM unnest($8::text[]) AS pfx + WHERE path = pfx + OR left(path, length(pfx) + 1) = pfx || '/' ) ) + ORDER BY id DESC + LIMIT $4 OFFSET $5", + w_id, + query.trigger_kind, + query.path, + per_page as i64, + offset as i64, + scope_all, + &scope_exact[..], + &scope_prefix[..], + ) + .fetch_all(&mut *tx) + .await?; + + tx.commit().await?; + + Ok(axum::Json(history)) +} diff --git a/backend/windmill-audit/src/lib.rs b/backend/windmill-audit/src/lib.rs index 8e74edfbf0..f1443fdb18 100644 --- a/backend/windmill-audit/src/lib.rs +++ b/backend/windmill-audit/src/lib.rs @@ -37,5 +37,8 @@ pub struct ListAuditLogQuery { pub resource: Option, pub before: Option>, pub after: Option>, + // Keyset cursor on the `id DESC` ordering. Lets a client stream a page in small batches + // without paying a growing OFFSET on every batch. + pub before_id: Option, pub all_workspaces: Option, } diff --git a/backend/windmill-common/src/external_ip.rs b/backend/windmill-common/src/external_ip.rs index b9713f67fd..5ffd713f5b 100644 --- a/backend/windmill-common/src/external_ip.rs +++ b/backend/windmill-common/src/external_ip.rs @@ -12,8 +12,58 @@ //! connections to be from whitelisted IP addresses. use crate::utils::configure_client; +use std::sync::OnceLock; use std::time::Duration; +/// No address has ever been established for the row. Matches the `worker_ping.ip` column default, +/// and doubles as what an agent sends while its lookup is in flight, since a server that predates +/// the lookup being asynchronous rejects an initial ping carrying nothing. +pub const UNKNOWN_IP: &str = "NO IP"; + +/// The lookup ran and could not produce an address. Distinct from [`UNKNOWN_IP`] because it tells +/// an operator the difference between "never asked" and "asked, and this instance cannot reach the +/// hub", which is the actionable one. Both are filtered out of the addresses the frontend offers +/// for whitelisting. +pub const UNRETRIEVABLE_IP: &str = "unretrievable IP"; + +/// `worker_ping.ip` is `VARCHAR(50)`, and a failed initial ping takes the worker down, so an +/// overlong value must not reach the insert. +const MAX_IP_LEN: usize = 50; + +static EXTERNAL_IP: OnceLock = OnceLock::new(); + +/// The external IP of this process, [`UNRETRIEVABLE_IP`] once the lookup has failed, or `None` +/// while it is still in flight. +pub fn cached_ip() -> Option<&'static str> { + EXTERNAL_IP.get().map(String::as_str) +} + +/// Resolves the external IP into the process-wide cache without blocking the caller. The value is +/// informational, and behind a firewall the lookup burns its whole 5s connect timeout on every +/// process start, so nothing on the worker startup path may wait on it. +pub fn resolve_ip_in_background() { + tokio::spawn(async { + let ip = get_ip() + .await + .map(|ip| { + if ip.len() > MAX_IP_LEN { + tracing::error!("external IP lookup returned an overlong value, ignoring it"); + UNRETRIEVABLE_IP.to_string() + } else { + ip + } + }) + .unwrap_or_else(|e| { + tracing::warn!( + error = e.to_string(), + "failed to get external IP, workers of this process will report it as unretrievable" + ); + UNRETRIEVABLE_IP.to_string() + }); + let _ = EXTERNAL_IP.set(ip); + }); +} + pub async fn get_ip() -> anyhow::Result { tokio::select! { biased; diff --git a/backend/windmill-common/src/feature_usage_oss.rs b/backend/windmill-common/src/feature_usage_oss.rs new file mode 100644 index 0000000000..3219e9d0ac --- /dev/null +++ b/backend/windmill-common/src/feature_usage_oss.rs @@ -0,0 +1,26 @@ +//! OSS fallback for anonymous feature-usage collection. +//! +//! Collection is a `private` feature (see `feature_usage_ee`). The public build +//! never sends a stats payload (`stats_oss`), so counting anything would only +//! write rows nothing reads: every entry point here is inert, and the +//! `log_feature_usage` endpoint accepts its posts without recording them. + +use sqlx::{Pool, Postgres}; + +/// No action is recordable in the public build. +pub fn is_recordable_event( + _feature: &str, + _kind: &str, + _key: &str, + _entity_id: &str, +) -> bool { + false +} + +/// No-op: nothing is counted in the public build. +pub fn log_feature_usage(_feature: &'static str, _kind: &'static str, _key: &str) {} + +/// Nothing accumulates, so there is nothing to flush. +pub async fn flush_feature_usage(_db: &Pool) -> Result<(), sqlx::Error> { + Ok(()) +} diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index 1a5cb3f1e6..37b4b60da4 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -331,6 +331,60 @@ use crate::error; use sqlx::postgres::Postgres; use sqlx::Pool; +/// Read several settings in one round trip. Names with no row are simply absent from the +/// result, exactly as [`load_value_from_global_settings`] returns `None` for them. +pub async fn load_values_from_global_settings( + db: &Pool, + names: &[&str], +) -> error::Result> { + // Listing the names keeps this on the primary key. `global_settings` also holds + // `workspace_dependencies_map_rebuilt:`, one row per workspace with no + // cleanup path, so a predicate that scanned the table would grow with workspace count. + let rows = sqlx::query!( + "SELECT name, value FROM global_settings WHERE name = ANY($1)", + names as &[&str] + ) + .fetch_all(db) + .await?; + Ok(rows.into_iter().map(|r| (r.name, r.value)).collect()) +} + +/// Return the instance's JWT secret, generating one only if the row holds nothing usable. +/// +/// The write has to be conditional rather than a plain upsert, for two reasons. A usable +/// secret must never be overwritten: replicas booting together would each install their own +/// and reject each other's tokens. And `notify_global_setting_change` fires on every write to +/// this table, so an unconditional upsert would make each startup trigger a cluster-wide +/// settings reload. An empty `RETURNING` is how a caller learns another process's secret +/// stands, and reads that one instead. +/// +/// Safe to call with a value read earlier: the statement, not the caller's read, decides. +pub async fn get_or_create_jwt_secret(db: &Pool) -> error::Result { + let candidate = crate::utils::rd_string(32); + let stored = sqlx::query_scalar!( + "INSERT INTO global_settings (name, value) VALUES ($1, $2) + ON CONFLICT (name) DO UPDATE SET value = EXCLUDED.value + WHERE jsonb_typeof(global_settings.value) <> 'string' + RETURNING value", + JWT_SECRET_SETTING, + serde_json::to_value(&candidate)? + ) + .fetch_optional(db) + .await?; + + match stored { + Some(_) => Ok(candidate), + None => load_value_from_global_settings(db, JWT_SECRET_SETTING) + .await? + .and_then(|v| serde_json::from_value::(v).ok()) + .ok_or_else(|| { + error::Error::InternalErr( + "jwt_secret conflicted but holds no usable value".to_string(), + ) + }), + } +} + pub async fn load_value_from_global_settings( db: &Pool, setting_name: &str, diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index a8de6d74c5..b72d0ae1d9 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -56,6 +56,13 @@ pub mod email_ee; pub mod email_oss; pub mod error; pub mod external_ip; +#[cfg(feature = "private")] +pub mod feature_usage_ee; +pub mod feature_usage_oss; +#[cfg(feature = "private")] +pub use feature_usage_ee as feature_usage; +#[cfg(not(feature = "private"))] +pub use feature_usage_oss as feature_usage; pub mod flow_conversations; pub mod flow_status; pub mod flows; @@ -90,6 +97,7 @@ pub mod otel_oss; #[cfg(feature = "private")] pub mod partition_ee; pub mod partition_oss; +pub mod per_minute_counter; #[cfg(feature = "private")] pub use partition_ee as partition; #[cfg(not(feature = "private"))] @@ -121,6 +129,7 @@ pub mod teams_ee; pub mod teams_oss; pub mod tracing_init; pub mod trashbin; +pub mod trigger_history; pub mod triggers; pub mod user_drafts; pub mod usernames; diff --git a/backend/windmill-common/src/login_rate_limit.rs b/backend/windmill-common/src/login_rate_limit.rs index aede8df2e4..8c60c30b88 100644 --- a/backend/windmill-common/src/login_rate_limit.rs +++ b/backend/windmill-common/src/login_rate_limit.rs @@ -1,31 +1,24 @@ use chrono::Utc; -use dashmap::DashMap; use hyper::StatusCode; -use std::sync::atomic::{AtomicI32, AtomicI64, AtomicU64, Ordering}; +use std::sync::atomic::{AtomicI32, AtomicI64, Ordering}; use std::sync::LazyLock; use crate::error::{Error, Result}; +use crate::per_minute_counter::PerMinuteCounter; use crate::worker::CLOUD_HOSTED; -const DEFAULT_PER_IP_LIMIT: i32 = 120; -const DEFAULT_PER_ACCOUNT_LIMIT: i32 = 30; +const DEFAULT_PER_IP_LIMIT: u32 = 120; +const DEFAULT_PER_ACCOUNT_LIMIT: u32 = 30; const DEFAULT_GLOBAL_LIMIT: i32 = 10000; -const EVICTION_INTERVAL: u64 = 256; -struct RateLimitEntry { - count: i32, - minute_bucket: i64, -} - -static IP_RATE_LIMIT: LazyLock> = LazyLock::new(DashMap::new); -static ACCOUNT_RATE_LIMIT: LazyLock> = LazyLock::new(DashMap::new); +static IP_RATE_LIMIT: LazyLock> = LazyLock::new(PerMinuteCounter::new); +static ACCOUNT_RATE_LIMIT: LazyLock> = + LazyLock::new(PerMinuteCounter::new); static GLOBAL_COUNT: AtomicI32 = AtomicI32::new(0); static GLOBAL_MINUTE: AtomicI64 = AtomicI64::new(0); -static EVICTION_COUNTER: AtomicU64 = AtomicU64::new(0); - -static PER_IP_LIMIT: LazyLock = LazyLock::new(|| { +static PER_IP_LIMIT: LazyLock = LazyLock::new(|| { std::env::var("LOGIN_RATE_LIMIT_PER_IP") .ok() .and_then(|v| v.parse().ok()) @@ -35,11 +28,11 @@ static PER_IP_LIMIT: LazyLock = LazyLock::new(|| { static PER_IP_LIMIT_EXPLICIT: LazyLock = LazyLock::new(|| { std::env::var("LOGIN_RATE_LIMIT_PER_IP") .ok() - .and_then(|v| v.parse::().ok()) + .and_then(|v| v.parse::().ok()) .is_some() }); -static PER_ACCOUNT_LIMIT: LazyLock = LazyLock::new(|| { +static PER_ACCOUNT_LIMIT: LazyLock = LazyLock::new(|| { std::env::var("LOGIN_RATE_LIMIT_PER_ACCOUNT") .ok() .and_then(|v| v.parse().ok()) @@ -49,7 +42,7 @@ static PER_ACCOUNT_LIMIT: LazyLock = LazyLock::new(|| { static PER_ACCOUNT_LIMIT_EXPLICIT: LazyLock = LazyLock::new(|| { std::env::var("LOGIN_RATE_LIMIT_PER_ACCOUNT") .ok() - .and_then(|v| v.parse::().ok()) + .and_then(|v| v.parse::().ok()) .is_some() }); @@ -86,57 +79,11 @@ pub fn extract_client_ip(headers: &axum::http::HeaderMap) -> Option { None } -fn maybe_evict(maps: &[&DashMap], current_minute: i64) { - let count = EVICTION_COUNTER.fetch_add(1, Ordering::Relaxed); - if count % EVICTION_INTERVAL == 0 { - for map in maps { - map.retain(|_, v| v.minute_bucket >= current_minute - 1); - } - } -} - -/// Atomically check the rate limit and increment the counter. Follows the -/// `public_app_rate_limit.rs` pattern — the DashMap entry lock is held across -/// both the check and the increment, preventing TOCTOU races. -fn check_and_increment( - map: &DashMap, - key: &str, - limit: i32, - current_minute: i64, -) -> Result<()> { - let mut entry = map - .entry(key.to_string()) - .or_insert(RateLimitEntry { count: 0, minute_bucket: current_minute }); - - if entry.minute_bucket != current_minute { - entry.count = 0; - entry.minute_bucket = current_minute; - } - - if entry.count >= limit { - return Err(Error::Generic( - StatusCode::TOO_MANY_REQUESTS, - "Too many login attempts. Please try again later.".to_string(), - )); - } - - entry.count += 1; - Ok(()) -} - -fn record_failure(map: &DashMap, key: &str) { - let current_minute = Utc::now().timestamp() / 60; - - let mut entry = map - .entry(key.to_string()) - .or_insert(RateLimitEntry { count: 0, minute_bucket: current_minute }); - - if entry.minute_bucket != current_minute { - entry.count = 1; - entry.minute_bucket = current_minute; - } else { - entry.count += 1; - } +fn too_many_attempts() -> Error { + Error::Generic( + StatusCode::TOO_MANY_REQUESTS, + "Too many login attempts. Please try again later.".to_string(), + ) } /// Called BEFORE authentication. Checks and increments global + per-IP counters. @@ -147,36 +94,30 @@ pub fn check_and_increment_login_attempt( headers: &axum::http::HeaderMap, email: &str, ) -> Result<()> { - let current_minute = Utc::now().timestamp() / 60; - maybe_evict(&[&IP_RATE_LIMIT, &ACCOUNT_RATE_LIMIT], current_minute); - - // Global limit: always on, uses atomics (single key, no need for DashMap) - check_and_increment_global(current_minute)?; + // Global limit: always on, uses atomics (single key, no need for a map) + check_and_increment_global()?; // Per-IP limit: CLOUD_HOSTED or explicit opt-in if *CLOUD_HOSTED || *PER_IP_LIMIT_EXPLICIT { if let Some(ip) = extract_client_ip(headers) { - check_and_increment(&IP_RATE_LIMIT, &ip, *PER_IP_LIMIT, current_minute)?; + if !IP_RATE_LIMIT.try_increment(ip, *PER_IP_LIMIT) { + return Err(too_many_attempts()); + } } } // Per-account check (read-only, does not increment — failures are recorded separately) if *CLOUD_HOSTED || *PER_ACCOUNT_LIMIT_EXPLICIT { - let entry = ACCOUNT_RATE_LIMIT.get(email); - if let Some(entry) = entry { - if entry.minute_bucket == current_minute && entry.count >= *PER_ACCOUNT_LIMIT { - return Err(Error::Generic( - StatusCode::TOO_MANY_REQUESTS, - "Too many login attempts. Please try again later.".to_string(), - )); - } + if ACCOUNT_RATE_LIMIT.count(email) >= *PER_ACCOUNT_LIMIT { + return Err(too_many_attempts()); } } Ok(()) } -fn check_and_increment_global(current_minute: i64) -> Result<()> { +fn check_and_increment_global() -> Result<()> { + let current_minute = Utc::now().timestamp() / 60; let stored_minute = GLOBAL_MINUTE.load(Ordering::Relaxed); if stored_minute != current_minute { // Minute rolled over — reset. Race here is benign: worst case two threads @@ -188,10 +129,7 @@ fn check_and_increment_global(current_minute: i64) -> Result<()> { let count = GLOBAL_COUNT.fetch_add(1, Ordering::Relaxed); if count >= *GLOBAL_LIMIT { - return Err(Error::Generic( - StatusCode::TOO_MANY_REQUESTS, - "Too many login attempts. Please try again later.".to_string(), - )); + return Err(too_many_attempts()); } Ok(()) @@ -201,6 +139,6 @@ fn check_and_increment_global(current_minute: i64) -> Result<()> { /// Per-account is only active on CLOUD_HOSTED or when LOGIN_RATE_LIMIT_PER_ACCOUNT is explicitly set. pub fn record_login_failure(email: &str) { if *CLOUD_HOSTED || *PER_ACCOUNT_LIMIT_EXPLICIT { - record_failure(&ACCOUNT_RATE_LIMIT, email); + ACCOUNT_RATE_LIMIT.increment(email.to_string()); } } diff --git a/backend/windmill-common/src/per_minute_counter.rs b/backend/windmill-common/src/per_minute_counter.rs new file mode 100644 index 0000000000..f52426c4d7 --- /dev/null +++ b/backend/windmill-common/src/per_minute_counter.rs @@ -0,0 +1,127 @@ +use dashmap::DashMap; +use std::borrow::Borrow; +use std::hash::Hash; +use std::sync::atomic::{AtomicU64, Ordering}; + +/// Calls between eviction sweeps. +const EVICTION_INTERVAL: u64 = 256; + +struct Bucket { + count: u32, + minute: i64, +} + +/// Events recorded per key within the current wall-clock minute, held in process memory. +/// +/// Counts are per process and reset on restart, so N servers raise any threshold built on +/// this to N times its configured value. Stale keys are swept as calls come in, which keeps the +/// map bounded without a background task. +pub struct PerMinuteCounter { + buckets: DashMap, + calls: AtomicU64, +} + +impl PerMinuteCounter { + pub fn new() -> Self { + Self { buckets: DashMap::new(), calls: AtomicU64::new(0) } + } + + /// Events recorded for `key` in the current minute, without recording one. A key never + /// seen, or last seen in an earlier minute, counts as zero. + pub fn count(&self, key: &Q) -> u32 + where + K: Borrow, + Q: Hash + Eq + ?Sized, + { + let minute = current_minute(); + self.buckets + .get(key) + .filter(|bucket| bucket.minute == minute) + .map_or(0, |bucket| bucket.count) + } + + /// Record one event and return the new count for the current minute. + pub fn increment(&self, key: K) -> u32 { + self.bump_at(key, None, current_minute()).0 + } + + /// Record one event unless `key` has already reached `limit` this minute. Returns false + /// when the limit was already reached, in which case nothing was recorded. + pub fn try_increment(&self, key: K, limit: u32) -> bool { + self.bump_at(key, Some(limit), current_minute()).1 + } + + /// Returns the count for `minute` and whether this call recorded an event. Takes the + /// minute rather than reading the clock so the rollover and eviction paths are testable. + fn bump_at(&self, key: K, limit: Option, minute: i64) -> (u32, bool) { + // The entry guard holds a lock on its DashMap shard, and the `retain` below takes + // every shard. Keeping the guard alive across that call deadlocks the caller, so this + // block is load-bearing: it must end before the sweep, not be flattened into the body. + let outcome = { + let mut bucket = self + .buckets + .entry(key) + .or_insert(Bucket { count: 0, minute }); + if bucket.minute != minute { + bucket.minute = minute; + bucket.count = 0; + } + if limit.is_some_and(|limit| bucket.count >= limit) { + (bucket.count, false) + } else { + bucket.count += 1; + (bucket.count, true) + } + }; + // Periodically drop what neither the current nor the previous minute can still need. + // Counts every call, refusals included: a key pinned at its limit must still drive + // sweeps, or a sustained burst of refusals would leave the map unbounded. + if self.calls.fetch_add(1, Ordering::Relaxed) % EVICTION_INTERVAL == 0 { + self.buckets.retain(|_, bucket| bucket.minute >= minute - 1); + } + outcome + } +} + +fn current_minute() -> i64 { + chrono::Utc::now().timestamp() / 60 +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn refuses_at_the_limit_without_recording() { + let counter = PerMinuteCounter::new(); + assert_eq!(counter.bump_at("a".to_string(), Some(2), 100), (1, true)); + assert_eq!(counter.bump_at("a".to_string(), Some(2), 100), (2, true)); + // A refused call must not record, or a caller held at the limit would keep inflating + // its own count and never recover within the minute. + assert_eq!(counter.bump_at("a".to_string(), Some(2), 100), (2, false)); + assert_eq!(counter.bump_at("a".to_string(), Some(2), 100), (2, false)); + } + + #[test] + fn resets_on_the_next_minute() { + let counter = PerMinuteCounter::new(); + counter.bump_at("a".to_string(), None, 100); + counter.bump_at("a".to_string(), None, 100); + assert_eq!(counter.bump_at("a".to_string(), None, 101), (1, true)); + } + + #[test] + fn sweep_drops_keys_older_than_the_previous_minute() { + let counter = PerMinuteCounter::new(); + counter.bump_at("stale".to_string(), None, 100); + counter.bump_at("previous".to_string(), None, 199); + // Sweeps fire every EVICTION_INTERVAL calls; drive the counter to the next one with + // filler events that all land in minute 200. + while counter.calls.load(Ordering::Relaxed) <= EVICTION_INTERVAL { + counter.bump_at("filler".to_string(), None, 200); + } + assert!(!counter.buckets.contains_key("stale")); + assert!(counter.buckets.contains_key("previous")); + assert!(counter.buckets.contains_key("filler")); + } +} diff --git a/backend/windmill-common/src/server.rs b/backend/windmill-common/src/server.rs index bd948a4f96..19da3222e9 100644 --- a/backend/windmill-common/src/server.rs +++ b/backend/windmill-common/src/server.rs @@ -1,6 +1,6 @@ use serde::{Deserialize, Serialize}; -use crate::{error, DB}; +use crate::{error, global_settings::SMTP_SETTING, DB}; #[derive(Serialize, Deserialize, PartialEq, Clone, Debug)] pub struct Smtp { @@ -27,13 +27,17 @@ pub struct SmtpConfigOpt { } pub async fn load_smtp_config(db: &DB) -> error::Result> { - let config: SmtpConfigOpt = - sqlx::query_scalar!("SELECT value FROM global_settings WHERE name = 'smtp_settings'",) - .fetch_optional(db) - .await? - .map(|x| serde_json::from_value(x).ok()) - .flatten() - .unwrap_or_default(); + let value = crate::global_settings::load_value_from_global_settings(db, SMTP_SETTING).await?; + Ok(parse_smtp_config(value)) +} + +/// The half of [`load_smtp_config`] after the read, so a batched settings pass can parse a +/// value it already fetched. +pub fn parse_smtp_config(value: Option) -> Option { + let config: SmtpConfigOpt = value + .map(|x| serde_json::from_value(x).ok()) + .flatten() + .unwrap_or_default(); let config_smtp = if let (Some(host), username, password) = (config.smtp_host, config.smtp_username, config.smtp_password) @@ -87,7 +91,7 @@ pub async fn load_smtp_config(db: &DB) -> error::Result> { tracing::warn!("SMTP not configured"); } - Ok(smtp) + smtp } impl Default for SmtpConfigOpt { diff --git a/backend/windmill-common/src/trigger_history.rs b/backend/windmill-common/src/trigger_history.rs new file mode 100644 index 0000000000..70c86b9ffd --- /dev/null +++ b/backend/windmill-common/src/trigger_history.rs @@ -0,0 +1,551 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Append-only history of schedule and trigger mutations (`trigger_history`). +//! +//! Every field of a row is derived by the server at write time: the caller +//! passes what it is doing, never who it claims to be or where it claims to +//! come from. **Who** (the authed username, or nobody for a server-initiated +//! change) and **what** (a field-level diff computed from the row before and +//! after the write) are derived by the server and cannot be forged. **From what +//! kind of client** ([`TriggerSource`]) is weaker on purpose: a first-party +//! client declares itself in a header, so it attributes rather than proves — +//! see [`TriggerSource::of_request`]. +//! +//! # What is recorded +//! +//! Authoring a single trigger through its own surface — create, update, delete, +//! enable/disable/suspend, restore from the trashbin, and the workspace-wide +//! default-handler override — plus the server disabling one after a failure. +//! **Adding a route that authors a trigger means adding a `record` call to it**; +//! nothing enforces that, because the alternative (a database trigger) cannot +//! see who or which client asked, and would fire on every listener ping. +//! +//! Deliberately outside that line, and not a gap to be closed one call site at a +//! time: +//! +//! - **Cascades of renaming or deleting something else** — a script/flow rename +//! rewriting `script_path` (`triggers::update_triggers_script_path`), a user +//! being removed rewriting ownership. The event belongs to the runnable or the +//! user, not to the trigger. +//! - **Workspace-level bulk operations** — archive, fork clone, cross-workspace +//! deploy. They move whole workspaces; a per-trigger row per path would say +//! nothing the workspace event does not. +//! - **Runtime housekeeping** — clearing `paused_until` / `error` after a run, +//! consumer-offset state (`reset_offset`, `server_id`), the managed +//! ducklake-maintenance schedule. The same category as the `server_id` and +//! `last_server_ping` columns the diff already drops. +//! +//! # The server-initiated disables: the disable wins +//! +//! When the server disables a trigger it could not run, two things want to be +//! true and cannot both be guaranteed: the trigger ends up disabled, and the +//! history says who disabled it. The disable wins, every time. +//! +//! A trigger left enabled reads as healthy while never firing again, and for a +//! flow schedule nothing comes back to retry — it arms its next occurrence when +//! the flow *starts*, so once the runnable is gone that code is never reached +//! again. Enabled-and-dead is silent; disabled-without-an-audit-row is not, and +//! the trigger's own `error` column still says why. +//! +//! So each writer puts the disabling `UPDATE` and the record in one +//! transaction, with only the insert inside a savepoint +//! ([`record_in_disable_tx`]). Both land on the same commit, and the trigger's +//! row lock is held across the pair — so the row cannot end up describing a +//! trigger deleted and recreated at that path in between. If the insert alone +//! fails it rolls back to the savepoint, the disable still commits, and the lost +//! row is reported to the workspace error handler and the critical alert +//! channel — loud, never silent. +//! +//! # Authorization contract +//! +//! None of the helpers here authorize anything: they take a connection and +//! write what they are given, exactly like `audit_log`. A caller must already +//! have authorized the mutation *and* performed it, and must derive `username` +//! from the request's `ApiAuthed` and `source` from +//! [`TriggerSource::of_request`] — never from anything the request body +//! carries. Reads are gated separately, by the RLS policies on the table and by +//! the token scopes the listing route checks. + +use sqlx::{Acquire, PgConnection}; + +use crate::error::Result; + +/// Header a first-party client sets to name itself. Only `cli`, `ui` and `api` +/// mean anything; any other value, and the header being absent, falls back to +/// what the credentials say. +pub const CLIENT_HEADER: &str = "x-windmill-client"; + +/// `trigger_kind` a schedule is recorded under. Triggers use their own +/// `TriggerCrud::TRIGGER_TYPE`. +pub const SCHEDULE_TRIGGER_KIND: &str = "schedule"; + +/// The kind of client a trigger mutation came from. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TriggerSource { + /// A browser session in the Windmill app. + Ui, + /// The `wmill` CLI (including the git-sync pull that shells out to it). + Cli, + /// A direct API call with a token: user scripts, CI, third-party clients. + Api, + /// No request at all: a worker or a trigger listener disabling something + /// after a failure. + Worker, +} + +impl TriggerSource { + pub fn as_str(&self) -> &'static str { + match self { + TriggerSource::Ui => "ui", + TriggerSource::Cli => "cli", + TriggerSource::Api => "api", + TriggerSource::Worker => "worker", + } + } + + fn from_client_header(value: &str) -> Option { + match value.trim().to_ascii_lowercase().as_str() { + "cli" => Some(TriggerSource::Cli), + "ui" => Some(TriggerSource::Ui), + "api" => Some(TriggerSource::Api), + _ => None, + } + } + + /// The source of the request currently being served. + /// + /// The declared client wins when it is one we know; otherwise the token + /// decides, and only the session token minted at browser login attributes + /// to the UI. Both inputs are attribution, never authority — nothing reads + /// a history row to make an access decision, so a caller lying about either + /// only mislabels its own row. + pub fn of_request(is_session_token: bool) -> Self { + match REQUEST_CLIENT.try_with(|client| *client) { + Ok(Some(source)) => source, + Ok(None) if is_session_token => TriggerSource::Ui, + Ok(None) => TriggerSource::Api, + // Outside a request there is no caller to attribute to. The + // server-initiated paths pass `Worker` themselves; this is what + // keeps a stray call from inventing one. + Err(_) => TriggerSource::Worker, + } + } +} + +/// What a mutation did to the trigger it is recorded against. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TriggerOperation { + Create, + Update, + Delete, + Enable, + Disable, + Suspend, +} + +impl TriggerOperation { + pub fn as_str(&self) -> &'static str { + match self { + TriggerOperation::Create => "create", + TriggerOperation::Update => "update", + TriggerOperation::Delete => "delete", + TriggerOperation::Enable => "enable", + TriggerOperation::Disable => "disable", + TriggerOperation::Suspend => "suspend", + } + } +} + +tokio::task_local! { + static REQUEST_CLIENT: Option; +} + +/// Run `f` with `client` as the declared client of every trigger mutation it +/// causes. Entered for every request, unmarked ones included, so that having no +/// scope at all means "not serving a request" — which is what +/// [`TriggerSource::Worker`] records. +pub async fn scope_client( + client: Option, + f: F, +) -> F::Output { + REQUEST_CLIENT.scope(client, f).await +} + +/// Parse the declared client of the request being served, if any. +pub fn client_from_header(value: &str) -> Option { + TriggerSource::from_client_header(value) +} + +/// Row fields that say nothing about the change itself: bookkeeping the history +/// row already carries, and listener runtime state that moves on its own. +const IGNORED_FIELDS: &[&str] = &[ + "workspace_id", + "edited_at", + "edited_by", + "extra_perms", + "last_server_ping", + "server_id", + // Listener runtime state like the two above: every trigger update clears it, + // so keeping it here would tag an ordinary edit with the failure it had + // before. The server-initiated disables put the error in `changes` + // themselves, so nothing is lost. + "error", + // Written from the requester on every schedule mutation, purely for workers + // that predate `permissioned_as`; it tracks the editor, not the schedule. + "email", +]; + +/// A `changes` payload bigger than this is replaced by the list of field names +/// it would have held. A schedule's `args` is caller-supplied and bounded only +/// by the API's request-size limit, and a history row is not worth a +/// multi-megabyte write. +const MAX_CHANGES_BYTES: usize = 32 * 1024; + +/// The row at `path` as JSON, or `None` when there is none — which, on an RLS +/// connection, also covers a row the caller cannot see. +/// +/// `FOR UPDATE`, so the preimage and the mutation that follows it see the same +/// row: without the lock another request can commit between the two, and its +/// change then lands in this caller's diff under this caller's name. +/// +/// Two things follow from taking the lock here rather than at the write: +/// +/// - The only row locked is the one the caller is about to write, and the +/// schedule paths reach the job queue only afterwards, so their documented +/// schedule-then-queue order is unchanged. +/// - The lock is held for whatever the caller does before its own `UPDATE`. For +/// `TriggerCrud::update_trigger` that includes the impl's external work — the +/// postgres impl opens a replication slot on a user-supplied host, the gcp and +/// azure impls call their subscription APIs — so a concurrent `setmode`, a +/// listener error write, or a script rename's bulk `script_path` update waits +/// on that call. Bounded by those APIs, not by us; the alternative is a +/// preimage inside each impl next to its own `UPDATE`. +/// +/// `table` is interpolated: pass a compile-time constant, never anything a +/// caller can reach. +pub async fn snapshot_row( + conn: &mut PgConnection, + table: &'static str, + workspace_id: &str, + path: &str, +) -> Result> { + // SAFETY: `table` is a compile-time constant. + let snapshot: Option = sqlx::query_scalar(&format!( + "SELECT to_jsonb(t) FROM {table} t WHERE workspace_id = $1 AND path = $2 FOR UPDATE" + )) + .bind(workspace_id) + .bind(path) + .fetch_optional(&mut *conn) + .await?; + Ok(snapshot) +} + +/// A field-level diff of two row snapshots, as `{field: {"old": …, "new": …}}`, +/// with `"old"` omitted where there is none to report. +/// +/// A create (`before` absent) keeps every non-null column of the new row, which +/// is its initial shape including whatever the column defaults supplied — +/// `to_jsonb` cannot tell a caller-set column from a defaulted one. Returns +/// `None` when nothing meaningful changed. +pub fn summarize_changes( + before: Option<&serde_json::Value>, + after: Option<&serde_json::Value>, +) -> Option { + let empty = serde_json::Map::new(); + let before = before.and_then(|v| v.as_object()).unwrap_or(&empty); + let after = after.and_then(|v| v.as_object())?; + + // Names of the changed fields, and the running size of what has been cloned + // so far. Measured as it goes rather than by serializing the finished map: + // a caller-sized `args` would otherwise be cloned in full and then copied + // again just to learn it was too big. + let mut fields = Vec::new(); + let mut changes = serde_json::Map::new(); + let mut bytes = 0usize; + for (field, new_value) in after { + if IGNORED_FIELDS.contains(&field.as_str()) { + continue; + } + let old_value = before.get(field); + match old_value { + Some(old_value) if old_value == new_value => continue, + None if new_value.is_null() => continue, + _ => {} + } + fields.push(field.clone()); + if bytes <= MAX_CHANGES_BYTES { + bytes += json_len(new_value) + old_value.map_or(0, json_len) + field.len(); + } + if bytes > MAX_CHANGES_BYTES { + continue; + } + let mut entry = serde_json::Map::new(); + if let Some(old_value) = old_value { + entry.insert("old".to_string(), old_value.clone()); + } + entry.insert("new".to_string(), new_value.clone()); + changes.insert(field.clone(), serde_json::Value::Object(entry)); + } + + if fields.is_empty() { + return None; + } + if bytes > MAX_CHANGES_BYTES { + return Some(serde_json::json!({ "truncated_fields": fields })); + } + Some(serde_json::Value::Object(changes)) +} + +/// Serialized size of `value` without building the string for it. +fn json_len(value: &serde_json::Value) -> usize { + struct Counter(usize); + impl std::io::Write for Counter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.0 += buf.len(); + Ok(buf.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let mut counter = Counter(0); + let _ = serde_json::to_writer(&mut counter, value); + counter.0 +} + +/// jsonb rejects `\u0000` inside a string, and `changes` quotes caller-supplied +/// text — a schedule's `args`, a worker's error message. One NUL anywhere in +/// there would fail the insert and cost the row. +fn strip_nuls(value: &mut serde_json::Value) { + match value { + serde_json::Value::String(s) if s.contains('\0') => *s = s.replace('\0', ""), + serde_json::Value::Array(items) => items.iter_mut().for_each(strip_nuls), + serde_json::Value::Object(map) => map.values_mut().for_each(strip_nuls), + _ => {} + } +} + +/// The last word on what reaches the column, applied at the write itself so a +/// hand-built `changes` (the server-initiated disables carry an error string of +/// unknown length and origin) gets it too, not just a computed diff. +fn cap_changes(changes: Option) -> Option { + let mut changes = changes?; + strip_nuls(&mut changes); + if json_len(&changes) <= MAX_CHANGES_BYTES { + return Some(changes); + } + let fields = changes + .as_object() + .map(|o| o.keys().cloned().collect::>()) + .unwrap_or_default(); + Some(serde_json::json!({ "truncated_fields": fields })) +} + +/// One trigger mutation, as it is about to be recorded. +#[derive(Clone)] +pub struct TriggerHistoryEvent<'a> { + pub workspace_id: &'a str, + /// `"schedule"`, or the trigger's `TRIGGER_TYPE` (`"http"`, `"kafka"`, …). + pub trigger_kind: &'a str, + pub path: &'a str, + pub operation: TriggerOperation, + pub source: TriggerSource, + /// `None` when the server acted on its own. + pub username: Option<&'a str>, + pub changes: Option, +} + +impl<'a> TriggerHistoryEvent<'a> { + /// The event for a trigger the server disabled on its own after a failure. + /// + /// `forced_state` is the column the disable wrote, in the same + /// `{field: {old, new}}` shape as a diff — the two disable paths write + /// different columns (`enabled` for a schedule, `mode` for a trigger). + /// + /// Record this only when the disabling `UPDATE` reported an affected row, + /// and only when that `UPDATE` was itself predicated on the trigger still + /// being enabled. The server reads the trigger long before it writes, so + /// without both the row describes a transition a user had already made. + pub fn server_disable( + workspace_id: &'a str, + trigger_kind: &'a str, + path: &'a str, + mut forced_state: serde_json::Value, + error: &str, + ) -> Self { + if let Some(obj) = forced_state.as_object_mut() { + obj.insert("error".to_string(), serde_json::json!({ "new": error })); + } + Self { + workspace_id, + trigger_kind, + path, + operation: TriggerOperation::Disable, + source: TriggerSource::Worker, + username: None, + changes: Some(forced_state), + } + } +} + +/// Record a disable inside the transaction that made it, without letting a +/// failed insert take the disable down with it. +/// +/// The caller's `UPDATE` holds the trigger's row lock until that transaction +/// commits, and this runs inside that window — so the row cannot end up +/// describing a trigger that was deleted and recreated at the same path in +/// between, which is the whole point of doing it here rather than on a second +/// connection afterwards. +/// +/// The insert itself goes in a savepoint. If it fails it rolls back alone, the +/// caller still commits the disable, and the reason comes back here so the +/// caller can alert: a trigger left enabled reads as healthy while never firing +/// again, which is worse than a missing audit row. +pub async fn record_in_disable_tx( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + event: TriggerHistoryEvent<'_>, +) -> Option { + let mut savepoint = match tx.begin().await { + Ok(savepoint) => savepoint, + Err(e) => return Some(e.to_string()), + }; + match record(&mut savepoint, event).await { + Ok(()) => savepoint.commit().await.err().map(|e| e.to_string()), + Err(e) => { + savepoint.rollback().await.ok(); + Some(e.to_string()) + } + } +} + +/// Append `event` to the history. +/// +/// Pass the same connection as the mutation for the two to commit together. +/// Does not authorize — see the module docs. +pub async fn record(conn: &mut PgConnection, event: TriggerHistoryEvent<'_>) -> Result<()> { + sqlx::query!( + "INSERT INTO trigger_history + (workspace_id, trigger_kind, path, operation, source, username, changes) + VALUES ($1, $2, $3, $4, $5, $6, $7)", + event.workspace_id, + event.trigger_kind, + event.path, + event.operation.as_str(), + event.source.as_str(), + event.username, + cap_changes(event.changes) as _, + ) + .execute(&mut *conn) + .await?; + Ok(()) +} + +/// Append one row per path, all describing the same change. +/// +/// For the workspace-wide operations that rewrite every schedule at once, where +/// a per-path diff would cost a snapshot per row and say the same thing each +/// time. Does not authorize — see the module docs. +pub async fn record_bulk( + conn: &mut PgConnection, + workspace_id: &str, + trigger_kind: &str, + paths: &[String], + operation: TriggerOperation, + source: TriggerSource, + username: Option<&str>, + changes: Option, +) -> Result<()> { + if paths.is_empty() { + return Ok(()); + } + sqlx::query!( + "INSERT INTO trigger_history + (workspace_id, trigger_kind, path, operation, source, username, changes) + SELECT $1, $2, p, $3, $4, $5, $6 FROM unnest($7::text[]) p", + workspace_id, + trigger_kind, + operation.as_str(), + source.as_str(), + username, + cap_changes(changes) as _, + paths, + ) + .execute(&mut *conn) + .await?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + /// The whole worker side of the attribution rests on this: a mutation made + /// outside a request records `worker` without each call site saying so. + #[tokio::test] + async fn client_is_absent_outside_a_request() { + assert_eq!(TriggerSource::of_request(false), TriggerSource::Worker); + assert_eq!( + scope_client(None, async { TriggerSource::of_request(true) }).await, + TriggerSource::Ui + ); + assert_eq!( + scope_client(None, async { TriggerSource::of_request(false) }).await, + TriggerSource::Api + ); + assert_eq!( + scope_client(Some(TriggerSource::Cli), async { + TriggerSource::of_request(true) + }) + .await, + TriggerSource::Cli + ); + } + + /// `error` and `edited_at` stand in for the whole ignore list: every trigger + /// update clears `error`, so without it an ordinary edit would carry the + /// failure the trigger had before it. + #[test] + fn diff_keeps_only_what_changed() { + let before = json!({"schedule": "0 0 * * *", "enabled": true, "edited_at": "a", "error": "boom"}); + let after = json!({"schedule": "0 1 * * *", "enabled": true, "edited_at": "b", "error": null}); + assert_eq!( + summarize_changes(Some(&before), Some(&after)), + Some(json!({"schedule": {"old": "0 0 * * *", "new": "0 1 * * *"}})) + ); + assert_eq!(summarize_changes(Some(&before), Some(&before)), None); + } + + #[test] + fn create_drops_null_columns_and_bookkeeping() { + let after = json!({"schedule": "0 0 * * *", "summary": null, "workspace_id": "w"}); + assert_eq!( + summarize_changes(None, Some(&after)), + Some(json!({"schedule": {"new": "0 0 * * *"}})) + ); + } + + /// A NUL reaching the column fails the insert, and `changes` quotes + /// caller-supplied text — so this is the difference between a recorded + /// disable and a lost one. + #[test] + fn nul_bytes_never_reach_the_column() { + let changes = cap_changes(Some(json!({ "error": { "new": "boom\u{0}tail" } }))); + assert_eq!(changes, Some(json!({ "error": { "new": "boomtail" } }))); + } + + #[test] + fn oversized_changes_keep_the_field_names() { + let after = json!({ "args": "x".repeat(MAX_CHANGES_BYTES + 1) }); + assert_eq!( + summarize_changes(None, Some(&after)), + Some(json!({"truncated_fields": ["args"]})) + ); + } +} diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index a94160be33..deff9c0588 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -30,6 +30,7 @@ use crate::{ agent_workers::PingJobStatusResponse, cache::{unwrap_or_error, RawNode, RawScript}, error::{self, to_anyhow}, + external_ip::UNKNOWN_IP, global_settings::CUSTOM_TAGS_SETTING, indexer::TantivyIndexerSettings, server::Smtp, @@ -948,20 +949,22 @@ pub fn write_file_at_user_defined_location( } pub async fn reload_custom_tags_setting(db: &DB) -> error::Result<()> { - let q = sqlx::query!( - "SELECT value FROM global_settings WHERE name = $1", - CUSTOM_TAGS_SETTING - ) - .fetch_optional(db) - .await?; + let q = + crate::global_settings::load_value_from_global_settings(db, CUSTOM_TAGS_SETTING).await?; + apply_custom_tags_setting(q); + Ok(()) +} +/// The half of [`reload_custom_tags_setting`] after the read, so a batched settings pass can +/// apply a value it already fetched. +pub fn apply_custom_tags_setting(q: Option) { let tags = if let Some(q) = q { - if let Ok(v) = serde_json::from_value::>(q.value.clone()) { + if let Ok(v) = serde_json::from_value::>(q.clone()) { v } else { tracing::error!( "Could not parse custom tags setting as vec of strings, found: {:#?}", - &q.value + &q ); vec![] } @@ -989,7 +992,6 @@ pub async fn reload_custom_tags_setting(db: &DB) -> error::Result<()> { ] .concat(), )); - Ok(()) } #[cfg(not(windows))] @@ -1749,25 +1751,23 @@ pub async fn update_ping_http( insert_ping.occupancy_rate_5m, insert_ping.occupancy_rate_30m, insert_ping.native_mode.unwrap_or(false), + insert_ping.ip.as_deref(), db, ) .await? } PingType::Initial => { - if insert_ping.worker_instance.is_none() - || insert_ping.version.is_none() - || insert_ping.ip.is_none() - { - return Err(anyhow::anyhow!( - "Worker instance, version and ip are required" - )); + if insert_ping.worker_instance.is_none() || insert_ping.version.is_none() { + return Err(anyhow::anyhow!("Worker instance and version are required")); } insert_ping_query( &insert_ping.worker_instance.unwrap(), &worker_name, worker_group, - &insert_ping.ip.unwrap(), + // An agent worker sends the sentinel rather than nothing, to stay acceptable to + // servers that still require an IP here; both mean "not resolved yet". + insert_ping.ip.as_deref().filter(|ip| *ip != UNKNOWN_IP), insert_ping.tags.unwrap_or_default().as_slice(), insert_ping.dw, insert_ping.dws.as_deref(), @@ -1904,12 +1904,13 @@ pub async fn fetch_raw_script_from_app_query( /// `wm_version`, hold the instance-wide `MIN_VERSION` back forever, and one still naming the /// job that process was killed mid-way through skews the zombie/OOM diagnostics that read it. /// `started_at` and `jobs_executed` are the only two columns carried over, being the -/// continuity itself. +/// continuity itself — plus `ip` for as long as `ip` is `None`, which means the external IP +/// lookup has not resolved yet and the predecessor's address is still the best guess. pub async fn insert_ping_query( worker_instance: &str, worker_name: &str, worker_group: &str, - ip: &str, + ip: Option<&str>, tags: &[String], dw: Option, dws: Option<&[String]>, @@ -1920,9 +1921,13 @@ pub async fn insert_ping_query( native_mode: bool, db: &DB, ) -> anyhow::Result { + // A NULL `ip` means the external IP lookup is still in flight; a later ping fills it in, and + // meanwhile the value a previous process wrote to a reclaimed row is the best guess we have. A + // lookup that has failed reports `external_ip::UNRETRIEVABLE_IP`, which does overwrite it. The + // literal below must stay equal to `external_ip::UNKNOWN_IP`. let previous_jobs_executed = sqlx::query_scalar!( - "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, dedicated_workers, wm_version, vcpus, memory, job_isolation, native_mode) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) ON CONFLICT (worker) - DO UPDATE set ping_at = now(), worker_instance = EXCLUDED.worker_instance, ip = EXCLUDED.ip, custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group, dedicated_worker = EXCLUDED.dedicated_worker, dedicated_workers = EXCLUDED.dedicated_workers, wm_version = EXCLUDED.wm_version, vcpus = COALESCE(EXCLUDED.vcpus, worker_ping.vcpus), memory = COALESCE(EXCLUDED.memory, worker_ping.memory), job_isolation = EXCLUDED.job_isolation, native_mode = EXCLUDED.native_mode, current_job_id = NULL, current_job_workspace_id = NULL + "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, dedicated_workers, wm_version, vcpus, memory, job_isolation, native_mode) VALUES ($1, $2, COALESCE($3, 'NO IP'), $4, $5, $6, $7, $8, $9, $10, $11, $12) ON CONFLICT (worker) + DO UPDATE set ping_at = now(), worker_instance = EXCLUDED.worker_instance, ip = COALESCE($3, worker_ping.ip), custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group, dedicated_worker = EXCLUDED.dedicated_worker, dedicated_workers = EXCLUDED.dedicated_workers, wm_version = EXCLUDED.wm_version, vcpus = COALESCE(EXCLUDED.vcpus, worker_ping.vcpus), memory = COALESCE(EXCLUDED.memory, worker_ping.memory), job_isolation = EXCLUDED.job_isolation, native_mode = EXCLUDED.native_mode, current_job_id = NULL, current_job_workspace_id = NULL RETURNING jobs_executed", worker_instance, worker_name, @@ -2027,12 +2032,13 @@ pub async fn update_worker_ping_main_loop_query( occupancy_rate_5m: Option, occupancy_rate_30m: Option, native_mode: bool, + ip: Option<&str>, db: &DB, ) -> anyhow::Result<()> { timeout(Duration::from_secs(10), sqlx::query!( "UPDATE worker_ping SET ping_at = now(), jobs_executed = $1, custom_tags = $2, occupancy_rate = $3, memory_usage = $4, wm_memory_usage = $5, vcpus = COALESCE($7, vcpus), - memory = COALESCE($8, memory), occupancy_rate_15s = $9, occupancy_rate_5m = $10, occupancy_rate_30m = $11, native_mode = $12 WHERE worker = $6", + memory = COALESCE($8, memory), occupancy_rate_15s = $9, occupancy_rate_5m = $10, occupancy_rate_30m = $11, native_mode = $12, ip = COALESCE($13, ip) WHERE worker = $6", jobs_executed, tags, occupancy_rate, @@ -2045,6 +2051,7 @@ pub async fn update_worker_ping_main_loop_query( occupancy_rate_5m, occupancy_rate_30m, native_mode, + ip, ) .execute(db)) .await??; diff --git a/backend/windmill-common/tests/global_settings_batch.rs b/backend/windmill-common/tests/global_settings_batch.rs new file mode 100644 index 0000000000..7701a9505a --- /dev/null +++ b/backend/windmill-common/tests/global_settings_batch.rs @@ -0,0 +1,94 @@ +//! `load_values_from_global_settings` is what a settings pass fetches with, so the difference +//! between "no row" and "the read failed" has to survive it: several settings reset to a +//! default when they read as unset, and would clobber a known-good value on a transient error. + +use serde_json::json; +use sqlx::{Pool, Postgres}; +use windmill_common::global_settings::{ + load_values_from_global_settings, BASE_URL_SETTING, SCIM_TOKEN_SETTING, +}; + +async fn set_setting(db: &Pool, name: &str, value: serde_json::Value) { + sqlx::query( + "INSERT INTO global_settings (name, value) VALUES ($1, $2) \ + ON CONFLICT (name) DO UPDATE SET value = EXCLUDED.value", + ) + .bind(name) + .bind(value) + .execute(db) + .await + .expect("failed to write global setting"); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn batch_returns_only_rows_that_exist(db: Pool) { + set_setting(&db, BASE_URL_SETTING, json!("set")).await; + // A dynamically named row, of which the table holds one per workspace with no cleanup + // path. Asking by name is what keeps a pass from scaling with how many of them exist. + set_setting(&db, "wm_test_dynamic:some_workspace", json!({})).await; + + let values = load_values_from_global_settings(&db, &[BASE_URL_SETTING, SCIM_TOKEN_SETTING]) + .await + .unwrap(); + + assert_eq!(values.get(BASE_URL_SETTING), Some(&json!("set"))); + assert_eq!( + values.get(SCIM_TOKEN_SETTING), + None, + "a name with no row must be absent, which the caller reads as unset" + ); + assert_eq!(values.len(), 1, "unrequested names must not come back"); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn batch_reports_failure_rather_than_an_empty_result(db: Pool) { + set_setting(&db, BASE_URL_SETTING, json!("set")).await; + + // Closing a pool makes every query on it fail. It has to be a separate pool built from the + // same options — `Pool` is a handle, so closing a clone of `db` would take `db` down too. + let unusable = sqlx::postgres::PgPoolOptions::new() + .connect_with((*db.connect_options()).clone()) + .await + .expect("failed to open second pool"); + unusable.close().await; + + assert!( + load_values_from_global_settings(&unusable, &[BASE_URL_SETTING]) + .await + .is_err(), + "a failed read must be an error, not an empty map that reads as every setting unset" + ); +} + +/// `get_or_create_jwt_secret` decides in SQL rather than from the caller's read, so that +/// replicas booting together cannot each install their own secret and reject each other's +/// tokens. Reverting it to a plain upsert would pass every other test in this file. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn jwt_secret_is_created_once_and_never_overwritten(db: Pool) { + let first = windmill_common::global_settings::get_or_create_jwt_secret(&db) + .await + .unwrap(); + assert!(!first.is_empty()); + + // Concurrent callers must converge on the one secret that landed, not clobber it. + let (a, b) = tokio::join!( + windmill_common::global_settings::get_or_create_jwt_secret(&db), + windmill_common::global_settings::get_or_create_jwt_secret(&db), + ); + assert_eq!(a.unwrap(), first); + assert_eq!(b.unwrap(), first); + + // A value that is not a usable secret is replaced rather than left in place. + set_setting(&db, "jwt_secret", json!(12345)).await; + let repaired = windmill_common::global_settings::get_or_create_jwt_secret(&db) + .await + .unwrap(); + assert_ne!(repaired, first); + assert_eq!( + windmill_common::global_settings::get_or_create_jwt_secret(&db) + .await + .unwrap(), + repaired, + "once repaired it must be stable again" + ); +} diff --git a/backend/windmill-oauth/src/lib.rs b/backend/windmill-oauth/src/lib.rs index 32648a2115..2650b68b4d 100644 --- a/backend/windmill-oauth/src/lib.rs +++ b/backend/windmill-oauth/src/lib.rs @@ -1179,6 +1179,18 @@ mod tests { assert!(verifier.verify("123", "body", "wrong_sig").is_err()); } + // Sage Intacct's token endpoint rejects HTTP Basic client authentication on the + // refresh_token grant (`invalid_client`), so its credentials must go in the form body. + #[test] + fn sage_intacct_registry_entry_uses_request_body_client_auth() { + let registry: HashMap = + serde_json::from_str(include_str!("../../oauth_connect.json")).unwrap(); + assert_eq!( + registry.get("sage_intacct").unwrap().req_body_auth, + Some(true) + ); + } + #[test] fn canonical_provider_name_strips_sandbox_suffix() { assert_eq!(canonical_provider_name("docusign_sandbox"), "docusign"); diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 2b4ebe8bfd..b286082b1c 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -2503,6 +2503,68 @@ pub async fn send_success_to_workspace_handler<'a, 'c, T: Serialize + Send + Syn Ok(()) } +/// The event for a schedule the server disabled on its own. +pub fn schedule_auto_disable_event<'a>( + workspace_id: &'a str, + path: &'a str, + error: &str, +) -> windmill_common::trigger_history::TriggerHistoryEvent<'a> { + windmill_common::trigger_history::TriggerHistoryEvent::server_disable( + workspace_id, + windmill_common::trigger_history::SCHEDULE_TRIGGER_KIND, + path, + serde_json::json!({ "enabled": { "old": true, "new": false } }), + error, + ) +} + +/// Disable a schedule the server can no longer arm, and record that it did. +/// +/// Contract on `record_in_disable_tx`. Here `tx` is the job-completion +/// transaction, so the savepoint also keeps a failed insert from poisoning it. +/// +/// Returns `Err` only when the disable itself failed; a lost history row comes +/// back through `history_lost` for the caller to report. +async fn disable_schedule_and_record( + tx: &mut Transaction<'_, Postgres>, + schedule: &Schedule, + err: &Error, + history_lost: &mut Option, +) -> Result { + let disable_result = sqlx::query!( + "UPDATE schedule SET enabled = false, error = $1 WHERE workspace_id = $2 AND path = $3 AND enabled = true", + err.to_string(), + &schedule.workspace_id, + &schedule.path + ) + .execute(&mut **tx) + .await; + + #[cfg(feature = "failpoints")] + let disable_result = if schedule_failpoints::is_active( + schedule_failpoints::ScheduleFailPoint::ScheduleDisable, + ) { + Err(sqlx::Error::Protocol( + "failpoint: schedule disable".to_string(), + )) + } else { + disable_result + }; + + let rows = disable_result?.rows_affected(); + // Zero rows means a user disabled the schedule first: no transition of ours + // to record. + if rows == 0 { + return Ok(0); + } + + let event = + schedule_auto_disable_event(&schedule.workspace_id, &schedule.path, &err.to_string()); + *history_lost = windmill_common::trigger_history::record_in_disable_tx(tx, event).await; + + Ok(rows) +} + pub async fn try_schedule_next_job<'c>( db: &Pool, mut tx: Transaction<'c, Postgres>, @@ -2657,36 +2719,31 @@ pub async fn try_schedule_next_job<'c>( "Could not push next scheduled job for {}: {err}. Disabling schedule.", schedule.path ); - let disable_result = sqlx::query!( - "UPDATE schedule SET enabled = false, error = $1 WHERE workspace_id = $2 AND path = $3", - err.to_string(), - &schedule.workspace_id, - &schedule.path - ) - .execute(&mut *tx) - .await; - #[cfg(feature = "failpoints")] - let disable_result = if schedule_failpoints::is_active( - schedule_failpoints::ScheduleFailPoint::ScheduleDisable, - ) { - Err(sqlx::Error::Protocol( - "failpoint: schedule disable".to_string(), - )) - } else { - disable_result - }; - if let Err(disable_err) = disable_result { + let mut history_lost = None; + match disable_schedule_and_record(&mut tx, schedule, err, &mut history_lost).await { + Err(disable_err) => { + report_error_to_workspace_handler_or_critical_side_channel( + job, + db, + format!( + "Could not push next scheduled job for {} and could not disable schedule: {disable_err}", + schedule.path, + ), + ) + .await; + } + Ok(_) => push_err = None, + } + if let Some(history_err) = history_lost { report_error_to_workspace_handler_or_critical_side_channel( job, db, format!( - "Could not push next scheduled job for {} and could not disable schedule: {disable_err}", + "Disabled schedule {} but could not record it in the trigger history: {history_err}", schedule.path, ), ) .await; - } else { - push_err = None; } } } @@ -6583,6 +6640,20 @@ async fn push_inner<'c, 'd>( ) .unzip(); + // Which trigger kinds an instance actually fires. Counted here rather than + // aggregated from `v2_job` later: that table's only usable index is + // (workspace_id, created_at), so a windowed GROUP BY over it is a full scan. + // + // Root jobs only. A scheduled flow hands every step push its own + // `schedule_path` (see `FlowJob::schedule_path`), so counting per push would + // score one run as a fire per step job — a loop pushes two of those per + // iteration — burying every other kind, and would sit on the per-step path. + if flow_step_id.is_none() { + if let Some(kind) = trigger_kind.as_ref() { + windmill_common::feature_usage::log_feature_usage("trigger", "fired", kind.as_str()); + } + } + #[cfg(feature = "cloud")] if *CLOUD_HOSTED { check_workspace_queue_cap(&mut *tx, workspace_id).await?; diff --git a/backend/windmill-store/src/resources.rs b/backend/windmill-store/src/resources.rs index a3b5d2a1a7..08ae1cfe5c 100644 --- a/backend/windmill-store/src/resources.rs +++ b/backend/windmill-store/src/resources.rs @@ -9,7 +9,6 @@ use dashmap::DashMap; use std::collections::HashMap; use std::net::IpAddr; -use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::LazyLock; use windmill_api_auth::{ @@ -17,6 +16,7 @@ use windmill_api_auth::{ require_super_admin, ApiAuthed, Tokened, }; use windmill_common::db::DB; +use windmill_common::per_minute_counter::PerMinuteCounter; use windmill_common::workspaces::{check_deploy_rules, RuleCheckResult}; use crate::secret_backend_ext::rename_vault_secret; @@ -1041,17 +1041,11 @@ pub const MAX_RESOURCE_VERSIONS: i64 = 100; /// low is the safe direction for something that only ever logs. const RESOURCE_WRITE_ADVISORY_PER_MIN: u32 = 20; -struct ResourceWriteRate { - count: u32, - minute_bucket: i64, -} - /// Writes seen per (workspace, path) per minute. Purely advisory, and deliberately so: nothing /// is throttled, the count is per process and resets on restart, so it undercounts across /// servers. That is affordable for a log line and is what keeps this off the write path proper. -static RESOURCE_WRITE_RATES: LazyLock> = - LazyLock::new(DashMap::new); -static RESOURCE_WRITES_SEEN: AtomicU64 = AtomicU64::new(0); +static RESOURCE_WRITE_RATES: LazyLock> = + LazyLock::new(PerMinuteCounter::new); /// Notice a caller rewriting one resource in a loop and point them at a store meant for it. /// Counts writes rather than versions: an unchanged value records nothing, but it still costs a @@ -1063,29 +1057,10 @@ fn note_resource_write(w_id: &str, path: &str, resource_type: &str) { if INTERNAL_RESOURCE_TYPES.contains(&resource_type) { return; } - let minute_bucket = chrono::Utc::now().timestamp() / 60; - // The entry guard holds a lock on its DashMap shard, and `retain` below takes every shard. - // Keeping the guard alive across that call deadlocks the request handler, so this block is - // load-bearing: it must end before the eviction, not be flattened into the function body. - let reached_cap = { - let mut rate = RESOURCE_WRITE_RATES - .entry((w_id.to_string(), path.to_string())) - .or_insert(ResourceWriteRate { count: 0, minute_bucket }); - if rate.minute_bucket != minute_bucket { - rate.minute_bucket = minute_bucket; - rate.count = 0; - } - rate.count += 1; - rate.count == RESOURCE_WRITE_ADVISORY_PER_MIN - }; - // Bounded without a background task: periodically drop what neither the current nor the - // previous minute can still need. - if RESOURCE_WRITES_SEEN.fetch_add(1, Ordering::Relaxed) % 256 == 0 { - RESOURCE_WRITE_RATES.retain(|_, rate| rate.minute_bucket >= minute_bucket - 1); - } + let writes = RESOURCE_WRITE_RATES.increment((w_id.to_string(), path.to_string())); // Once per minute per path: `==` rather than `>=` so a sustained loop logs at the crossing // and then stays quiet until the bucket rolls over. - if reached_cap { + if writes == RESOURCE_WRITE_ADVISORY_PER_MIN { tracing::warn!( workspace_id = %w_id, path = %path, diff --git a/backend/windmill-test-utils/src/lib.rs b/backend/windmill-test-utils/src/lib.rs index 060e8af983..c09ad2147d 100644 --- a/backend/windmill-test-utils/src/lib.rs +++ b/backend/windmill-test-utils/src/lib.rs @@ -442,7 +442,6 @@ pub fn spawn_test_worker( let (tx, rx) = KillpillSender::new(1); let worker_instance: &str = "test worker instance"; let worker_name: String = next_worker_name(); - let ip: &str = Default::default(); let conn = conn.to_owned(); let tx2 = tx.clone(); @@ -465,7 +464,6 @@ pub fn spawn_test_worker( worker_name, 1, 1, - ip, rx, tx2, &base_internal_url, @@ -494,7 +492,6 @@ pub fn spawn_test_worker_dedicated( let (tx, rx) = KillpillSender::new(1); let worker_instance: &str = "test worker instance"; let worker_name: String = next_worker_name(); - let ip: &str = Default::default(); let conn = conn.to_owned(); let tx2 = tx.clone(); @@ -548,7 +545,6 @@ pub fn spawn_test_worker_dedicated( worker_name, 1, 1, - ip, rx, tx2, &base_internal_url, diff --git a/backend/windmill-trigger-http/src/handler.rs b/backend/windmill-trigger-http/src/handler.rs index 17321b7775..14e1b9a4d2 100644 --- a/backend/windmill-trigger-http/src/handler.rs +++ b/backend/windmill-trigger-http/src/handler.rs @@ -307,6 +307,37 @@ pub async fn create_many_http_triggers( .map_err(|err| error_wrapper(&new_http_trigger.config.route_path, err.into()))?; } + // Bulk create is still authoring, so it records like the single-create + // route rather than being the one way to make a trigger appear with no + // history behind it. + let created = windmill_common::trigger_history::snapshot_row( + &mut *tx, + "http_trigger", + &w_id, + &new_http_trigger.base.path, + ) + .await + .map_err(|err| error_wrapper(&new_http_trigger.config.route_path, err))?; + windmill_common::trigger_history::record( + &mut *tx, + windmill_common::trigger_history::TriggerHistoryEvent { + workspace_id: &w_id, + trigger_kind: HttpTrigger::TRIGGER_TYPE, + path: &new_http_trigger.base.path, + operation: windmill_common::trigger_history::TriggerOperation::Create, + source: windmill_common::trigger_history::TriggerSource::of_request( + authed.is_session_token, + ), + username: Some(&authed.username), + changes: windmill_common::trigger_history::summarize_changes( + None, + created.as_ref(), + ), + }, + ) + .await + .map_err(|err| error_wrapper(&new_http_trigger.config.route_path, err))?; + audit_log( &mut *tx, &authed, diff --git a/backend/windmill-trigger/src/handler.rs b/backend/windmill-trigger/src/handler.rs index ad3a6b097a..00d8cbf828 100644 --- a/backend/windmill-trigger/src/handler.rs +++ b/backend/windmill-trigger/src/handler.rs @@ -16,6 +16,7 @@ use windmill_api_auth::{build_scope_path_predicate, check_scopes, ApiAuthed}; use windmill_common::{ db::UserDB, error::{Error, JsonResult, Result}, + trigger_history::{self, TriggerHistoryEvent, TriggerOperation, TriggerSource}, user_drafts::{ delete_all_drafts_for_path, delete_own_draft_for_path, fetch_draft_only_list_rows, overlay_or_draft_only, UserDraftItemKind, WithDraftOverlay, WithDraftQuery, @@ -458,6 +459,45 @@ pub trait TriggerCrud: Send + Sync + 'static { } } +/// Append this mutation to `trigger_history`, diffing the row at `path` against +/// `before`. +/// +/// Call it on the transaction that made the change, after the change: the +/// snapshot it takes is the "after" side of the diff, and the two commit or roll +/// back together. +/// +/// Records nothing when the snapshots say no row was written. `TriggerCrud::update_trigger` +/// returns `Result<()>` and several impls do not check `rows_affected`, so an +/// update aimed at a path that does not exist — or that RLS hides from the +/// caller — reaches here having changed nothing; without this the caller could +/// forge history rows at any path, since the insert policy is `WITH CHECK (true)`. +async fn record_trigger_history( + tx: &mut PgConnection, + authed: &ApiAuthed, + workspace_id: &str, + path: &str, + operation: TriggerOperation, + before: Option, +) -> Result<()> { + let after = trigger_history::snapshot_row(&mut *tx, T::TABLE_NAME, workspace_id, path).await?; + if after.is_none() || (operation == TriggerOperation::Update && before.is_none()) { + return Ok(()); + } + trigger_history::record( + &mut *tx, + TriggerHistoryEvent { + workspace_id, + trigger_kind: T::TRIGGER_TYPE, + path, + operation, + source: TriggerSource::of_request(authed.is_session_token), + username: Some(&authed.username), + changes: trigger_history::summarize_changes(before.as_ref(), after.as_ref()), + }, + ) + .await +} + pub fn trigger_routes() -> Router { let mut router = Router::new() .route("/create", post(create_trigger::)) @@ -556,6 +596,16 @@ async fn create_trigger( .await?; } + record_trigger_history::( + &mut *tx, + &authed, + &workspace_id, + &new_path, + TriggerOperation::Create, + None, + ) + .await?; + audit_log( &mut *tx, &authed, @@ -782,6 +832,9 @@ async fn update_trigger( &authed.username, ); + let before = + trigger_history::snapshot_row(&mut *tx, T::TABLE_NAME, &workspace_id, path).await?; + handler .update_trigger(&db, &mut *tx, &authed, &workspace_id, path, edit_trigger) .await?; @@ -799,6 +852,18 @@ async fn update_trigger( .await?; } + // Recorded at the new path, so a rename reads as one event there with + // `path` among the changed fields rather than a delete plus a create. + record_trigger_history::( + &mut *tx, + &authed, + &workspace_id, + &new_path, + TriggerOperation::Update, + before, + ) + .await?; + audit_log( &mut *tx, &authed, @@ -913,6 +978,22 @@ async fn delete_trigger( .await?; } + // No diff: the row is gone, and the trashbin above already keeps its full + // contents for a restore. + trigger_history::record( + &mut *tx, + TriggerHistoryEvent { + workspace_id: &workspace_id, + trigger_kind: T::TRIGGER_TYPE, + path, + operation: TriggerOperation::Delete, + source: TriggerSource::of_request(authed.is_session_token), + username: Some(&authed.username), + changes: None, + }, + ) + .await?; + audit_log( &mut *tx, &authed, @@ -1052,6 +1133,9 @@ async fn set_trigger_mode( } } + let before = + trigger_history::snapshot_row(&mut *tx, T::TABLE_NAME, &workspace_id, path).await?; + let updated = handler .set_trigger_mode(&authed, &mut *tx, &workspace_id, path, &payload.mode) .await?; @@ -1063,6 +1147,20 @@ async fn set_trigger_mode( ))); } + record_trigger_history::( + &mut *tx, + &authed, + &workspace_id, + path, + match payload.mode { + TriggerMode::Enabled => TriggerOperation::Enable, + TriggerMode::Disabled => TriggerOperation::Disable, + TriggerMode::Suspended => TriggerOperation::Suspend, + }, + before, + ) + .await?; + tx.commit().await?; handle_deployment_metadata( diff --git a/backend/windmill-trigger/src/listener.rs b/backend/windmill-trigger/src/listener.rs index e327ed3155..d2c80b4d65 100644 --- a/backend/windmill-trigger/src/listener.rs +++ b/backend/windmill-trigger/src/listener.rs @@ -386,9 +386,14 @@ pub trait Listener: TriggerCrud + TriggerJobArgs { error: String, ) { if listening_trigger.trigger_mode { - // SAFETY: Self::TABLE_NAME is a compile-time constant. - let report_status = sqlx::query(&format!( - r#" + // Contract on `record_in_disable_tx`: one transaction so the row + // lock spans both writes. + let mut history_err = None; + let report_status = async { + let mut tx = db.begin().await?; + // SAFETY: Self::TABLE_NAME is a compile-time constant. + let rows = sqlx::query(&format!( + r#" UPDATE {} SET @@ -398,18 +403,60 @@ pub trait Listener: TriggerCrud + TriggerJobArgs { last_server_ping = NULL WHERE workspace_id = $2 AND - path = $3 + path = $3 AND + mode <> 'disabled'::TRIGGER_MODE "#, - Self::TABLE_NAME - )) - .bind(&error) - .bind(&listening_trigger.workspace_id) - .bind(&listening_trigger.path) - .execute(db) + Self::TABLE_NAME + )) + .bind(&error) + .bind(&listening_trigger.workspace_id) + .bind(&listening_trigger.path) + .execute(&mut *tx) + .await? + .rows_affected(); + + // Zero rows: deleted, or a user disabled it first — no + // transition of ours to record. + if rows > 0 { + // `to_key`, not `Display`: it is what lines up with the + // `TRIGGER_TYPE` the API records under. + let trigger_kind = Self::TRIGGER_KIND.to_key(); + history_err = windmill_common::trigger_history::record_in_disable_tx( + &mut tx, + windmill_common::trigger_history::TriggerHistoryEvent::server_disable( + &listening_trigger.workspace_id, + &trigger_kind, + &listening_trigger.path, + serde_json::json!({ "mode": { "new": "disabled" } }), + &error, + ), + ) + .await; + } + tx.commit().await?; + Ok::<(), Error>(()) + } .await; + if let Some(history_err) = history_err { + // Spawned: the commit above made the cleared `server_id` visible, + // so the ping branch of the enclosing `select!` is about to + // finish and drop everything left in this future. Awaiting the + // alert here would lose the one signal that the row is missing. + let message = format!( + "Disabled {} trigger {} but could not record it in the trigger history: {}", + Self::TRIGGER_KIND, + listening_trigger.path, + history_err + ); + let (db, workspace_id) = (db.clone(), listening_trigger.workspace_id.clone()); + tokio::spawn(async move { + report_critical_error(message, db, Some(&workspace_id), None).await; + }); + } + match report_status { - Ok(_) => { + Ok(()) => { report_critical_error( format!( "Disabling {} trigger {} because of error: {}", diff --git a/backend/windmill-worker/src/python_versions.rs b/backend/windmill-worker/src/python_versions.rs index 11f673e8f4..11674efcd6 100644 --- a/backend/windmill-worker/src/python_versions.rs +++ b/backend/windmill-worker/src/python_versions.rs @@ -3,10 +3,12 @@ use std::{ process::Stdio, str::FromStr, sync::Arc, + time::UNIX_EPOCH, }; use chrono::{DateTime, Duration, Utc}; use itertools::Itertools; +use serde::{Deserialize, Serialize}; use serde_json::Value; use tokio::{fs::DirBuilder, process::Command, sync::RwLock}; use uuid::Uuid; @@ -465,7 +467,7 @@ impl PyV { w_id: &str, occupancy_metrics: &mut Option<&mut OccupancyMetrics>, ) -> error::Result> { - let py_path = self.find_python().await; + let py_path = self.find_python_cached().await; // Runtime is not installed if let Err(py_err) = py_path { @@ -480,7 +482,7 @@ impl PyV { return Err(err); } else { // Try to find one more time - let py_path = self.find_python().await; + let py_path = self.find_python_cached().await; if let Err(err) = py_path { tracing::error!( @@ -489,7 +491,6 @@ impl PyV { return Err(err); } - // TODO: Cache the result py_path } } else { @@ -600,7 +601,37 @@ impl PyV { .await?; Ok(()) } + /// Same as [`Self::find_python`] but backed by [`PY_PATH_CACHE_DIR`], which outlives the + /// worker process. The subprocess is only spawned when there is nothing usable on disk. + async fn find_python_cached(&self) -> error::Result> { + // Keyed on the requested version, not on the resolved patch: uv answers a minor-only + // request with its own minor-version link, which it re-points when a newer patch is + // installed, so an entry follows patch upgrades without being invalidated. + let version = self.to_string(); + // Without an identity for uv an upgrade would go unnoticed, so the cache is skipped. + let uv = uv_identity().await; + + if let Some(ref uv) = uv { + if let Some(py_path) = read_cached_python_path(&PY_PATH_CACHE_DIR, uv, &version).await { + // Serving a path that no longer exists is far worse than the spawn it saves, so + // the interpreter is checked instead of trusted (the install dir may have been + // wiped, or uv may have moved it). + if tokio::fs::try_exists(&py_path).await.unwrap_or(false) { + return Ok(Some(py_path)); + } + } + } + + let py_path = self.find_python().await; + if let (Some(uv), Ok(Some(py_path))) = (&uv, &py_path) { + write_cached_python_path(&PY_PATH_CACHE_DIR, uv, &version, py_path).await; + } + py_path + } + async fn find_python(&self) -> error::Result> { + tracing::debug!("Resolving python {} with uv python find", self.to_string()); + #[cfg(windows)] let uv_cmd = "uv"; @@ -670,6 +701,81 @@ impl PyV { } } +lazy_static::lazy_static! { + /// Sits next to `PY_INSTALL_DIR` rather than inside it, so uv never sees these entries while + /// scanning that directory for managed interpreters. + static ref PY_PATH_CACHE_DIR: String = format!("{}_paths", *PY_INSTALL_DIR); +} + +#[cfg(windows)] +lazy_static::lazy_static! { + /// uv is invoked as a bare `uv` on windows, hence resolved through PATH, which + /// [`tokio::fs::metadata`] does not search. PATH does not change under us, so the lookup is + /// done once. + static ref UV_PATH: Option = std::env::split_paths(PATH_ENV.as_str()) + .map(|dir| dir.join("uv.exe")) + .find(|path| path.is_file()) + .map(|path| path.to_string_lossy().into_owned()); +} + +/// Interpreter path resolved by `uv python find` for one requested version. +#[derive(Serialize, Deserialize)] +struct CachedPythonPath { + /// Identity of the uv that resolved `path`. An upgraded uv may pick a different interpreter + /// for the same request, so an entry left by another uv is ignored. + uv: String, + path: String, +} + +/// `None` disables the cache: an upgrade of a uv we cannot stat would go unnoticed. +async fn uv_identity() -> Option { + #[cfg(unix)] + let uv_cmd = UV_PATH.clone(); + + // Initializing the static probes PATH synchronously, which must not happen on the runtime. + #[cfg(windows)] + let uv_cmd = tokio::task::spawn_blocking(|| UV_PATH.clone()) + .await + .ok() + .flatten()?; + + let metadata = tokio::fs::metadata(&uv_cmd).await.ok()?; + let mtime = metadata.modified().ok()?.duration_since(UNIX_EPOCH).ok()?; + Some(format!("{uv_cmd}:{}:{}", metadata.len(), mtime.as_secs())) +} + +/// One file per version, so that workers resolving different versions concurrently cannot drop +/// each other's entry the way a shared map would. +fn cached_python_path_file(dir: &str, version: &str) -> String { + format!("{dir}/{version}.json") +} + +async fn read_cached_python_path(dir: &str, uv: &str, version: &str) -> Option { + let content = tokio::fs::read(cached_python_path_file(dir, version)) + .await + .ok()?; + let cached = serde_json::from_slice::(&content).ok()?; + (cached.uv == uv).then_some(cached.path) +} + +async fn write_cached_python_path(dir: &str, uv: &str, version: &str, py_path: &str) { + let cached = CachedPythonPath { uv: uv.to_owned(), path: py_path.to_owned() }; + + // Written aside and renamed so that a concurrent worker never reads a half-written entry. + let tmp_file = format!("{dir}/{}.tmp", Uuid::new_v4()); + let write = async { + tokio::fs::create_dir_all(dir).await?; + tokio::fs::write(&tmp_file, serde_json::to_vec(&cached)?).await?; + tokio::fs::rename(&tmp_file, cached_python_path_file(dir, version)).await?; + Ok::<_, anyhow::Error>(()) + }; + + if let Err(e) = write.await { + tracing::warn!("Could not cache resolved python path ({py_path}): {e}"); + let _ = tokio::fs::remove_file(&tmp_file).await; + } +} + #[cfg(test)] mod tests { use super::*; @@ -965,4 +1071,25 @@ mod tests { ) .await; } + + #[tokio::test] + async fn test_cached_python_path_is_scoped_to_uv() { + let dir = std::env::temp_dir() + .join(format!("wm_py_path_cache_{}", Uuid::new_v4())) + .to_string_lossy() + .into_owned(); + + write_cached_python_path(&dir, "uv-a", "3.12", "/py/3.12/bin/python3.12").await; + assert_eq!( + read_cached_python_path(&dir, "uv-a", "3.12") + .await + .as_deref(), + Some("/py/3.12/bin/python3.12") + ); + // An upgraded uv may pick a different interpreter, so its entries cannot be reused + assert_eq!(read_cached_python_path(&dir, "uv-b", "3.12").await, None); + assert_eq!(read_cached_python_path(&dir, "uv-a", "3.13").await, None); + + tokio::fs::remove_dir_all(&dir).await.unwrap(); + } } diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index eb35b38d4c..090d37bcab 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -41,6 +41,7 @@ use windmill_common::{ agent_workers::DECODED_AGENT_TOKEN, apps::AppScriptId, cache::{future::FutureCachedExt, ScriptData, ScriptMetadata}, + external_ip::cached_ip, schema::{should_validate_schema, SchemaValidator}, utils::{create_directory_async, WarnAfterExt}, worker::{ @@ -261,6 +262,7 @@ const NUM_SECS_READINGS: u64 = 60; const INCLUDE_DEPS_PY_SH_CONTENT: &str = include_str!("../nsjail/download_deps.py.sh"); const WORKER_SHELL_NAP_TIME_DURATION: u64 = 15; +const WORKER_SHELL_INITIAL_NAP_TIME_DURATION: u64 = 5; const TIMEOUT_TO_RESET_WORKER_SHELL_NAP_TIME_DURATION: u64 = 2 * 60; pub const DEFAULT_SLEEP_QUEUE: u64 = 50; @@ -2201,6 +2203,71 @@ pub async fn handle_all_job_kind_error( } } +/// How long the interactive shell loop waits before polling its tag again, when it found no +/// job. The sub-second cadence only pays off while somebody is typing into the shell, so it +/// is reserved for a session that has run a command: before the first one this process serves +/// there is nothing to keep responsive, only the next session to notice. +/// +/// - a live session, last command under `TIMEOUT_TO_RESET_WORKER_SHELL_NAP_TIME_DURATION` +/// ago: `sleep_queue() * 10` +/// - no command yet this process: `WORKER_SHELL_INITIAL_NAP_TIME_DURATION`, which bounds how +/// long the first command of a session waits +/// - nothing for `TIMEOUT_TO_RESET_WORKER_SHELL_NAP_TIME_DURATION`, counted from the last +/// command or from process start: `WORKER_SHELL_NAP_TIME_DURATION` +/// +/// A worker whose process is recycled after N jobs cannot count on living long enough to +/// reach that last state, and at N=1 never does, so it starts there instead. That holds for +/// any N, since N says nothing about how long a process lasts. +fn interactive_shell_nap( + now: Instant, + started_at: Instant, + last_executed_job: Option, + recycles_after_n_jobs: bool, +) -> Duration { + let quiet_since = last_executed_job.unwrap_or(started_at); + if now.duration_since(quiet_since).as_secs() > TIMEOUT_TO_RESET_WORKER_SHELL_NAP_TIME_DURATION { + return Duration::from_secs(WORKER_SHELL_NAP_TIME_DURATION); + } + match last_executed_job { + Some(_) => Duration::from_millis(sleep_queue() * 10), + None if recycles_after_n_jobs => Duration::from_secs(WORKER_SHELL_NAP_TIME_DURATION), + None => Duration::from_secs(WORKER_SHELL_INITIAL_NAP_TIME_DURATION), + } +} + +#[cfg(test)] +mod interactive_shell_nap_tests { + use super::*; + + const LONG: Duration = Duration::from_secs(WORKER_SHELL_NAP_TIME_DURATION); + const INITIAL: Duration = Duration::from_secs(WORKER_SHELL_INITIAL_NAP_TIME_DURATION); + + #[test] + fn only_a_live_shell_session_gets_the_sub_second_cadence() { + let start = Instant::now(); + let quiet = + start + Duration::from_secs(TIMEOUT_TO_RESET_WORKER_SHELL_NAP_TIME_DURATION + 1); + let fast = Duration::from_millis(sleep_queue() * 10); + // Nobody has opened a shell on this worker yet, so there is no session to keep + // responsive: only the first command of the next one to notice. + assert_eq!(interactive_shell_nap(start, start, None, false), INITIAL); + assert_eq!(interactive_shell_nap(quiet, start, None, false), LONG); + // A worker recycled after N jobs may never live to back off, so it starts backed off. + assert_eq!(interactive_shell_nap(start, start, None, true), LONG); + // Either way, a served shell job is a live session and gets the fast cadence. + assert_eq!(interactive_shell_nap(quiet, start, Some(quiet), true), fast); + assert_eq!( + interactive_shell_nap( + quiet + Duration::from_secs(TIMEOUT_TO_RESET_WORKER_SHELL_NAP_TIME_DURATION + 1), + start, + Some(quiet), + true + ), + LONG + ); + } +} + fn start_interactive_worker_shell( conn: Connection, hostname: String, @@ -2211,10 +2278,12 @@ fn start_interactive_worker_shell( worker_dir: String, ) -> JoinHandle<()> { tokio::spawn(async move { - let mut occupancy_metrics = OccupancyMetrics::new(Instant::now()); + let started_at = Instant::now(); + let mut occupancy_metrics = OccupancyMetrics::new(started_at); - let mut last_executed_job: Option = - Instant::now().checked_sub(Duration::from_millis(2500)); + // `None` means no shell job has been served yet, which the nap distinguishes from a + // shell session that has gone quiet. + let mut last_executed_job: Option = None; loop { if let Ok(_) = killpill_rx.try_recv() { @@ -2329,16 +2398,12 @@ fn start_interactive_worker_shell( last_executed_job = Some(Instant::now()); } Ok(None) => { - let now = Instant::now(); - let nap_time = match last_executed_job { - Some(last) - if now.duration_since(last).as_secs() - > TIMEOUT_TO_RESET_WORKER_SHELL_NAP_TIME_DURATION => - { - Duration::from_secs(WORKER_SHELL_NAP_TIME_DURATION) - } - _ => Duration::from_millis(sleep_queue() * 10), - }; + let nap_time = interactive_shell_nap( + Instant::now(), + started_at, + last_executed_job, + EXIT_AFTER_N_JOBS.is_some(), + ); tokio::select! { _ = tokio::time::sleep(nap_time) => { } @@ -2487,7 +2552,6 @@ pub async fn run_worker( worker_name: String, i_worker: u64, num_workers: u32, - ip: &str, mut killpill_rx: tokio::sync::broadcast::Receiver<()>, killpill_tx: KillpillSender, base_internal_url: &str, @@ -2583,7 +2647,8 @@ pub async fn run_worker( let mut last_ping = Instant::now() - Duration::from_secs(NUM_SECS_PING + 1); - let previous_jobs_executed = insert_ping(hostname, &worker_name, ip, conn) + let mut reported_ip = cached_ip(); + let previous_jobs_executed = insert_ping(hostname, &worker_name, reported_ip, conn) .await .expect("initial ping could be sent"); @@ -2799,7 +2864,7 @@ pub async fn run_worker( .is_some_and(|dws| !dws.is_empty()) }; - if EXIT_AFTER_N_JOBS.is_some() && i_worker == 1 { + if let Some(max_jobs) = (*EXIT_AFTER_N_JOBS).filter(|_| i_worker == 1) { if num_workers > 1 { tracing::warn!( worker = %worker_name, hostname = %hostname, @@ -2816,6 +2881,27 @@ pub async fn run_worker( workers: those run outside its main loop and are never counted." ); } + let config = WORKER_CONFIG.load(); + if config.init_bash.is_some() { + tracing::warn!( + worker = %worker_name, hostname = %hostname, + "EXIT_AFTER_N_JOBS is set and this worker group has an init script: the init \ + script prepares the environment the limit recycles, so it is not counted and runs \ + again on every restart. Every {max_jobs} job(s) therefore pushes and executes an \ + init job of its own first, and waits for it." + ); + } + // No interval check: loading a worker config whose periodic script has no interval, or + // one below MIN_PERIODIC_SCRIPT_INTERVAL_SECONDS, fails and kills the worker, so a + // script that reaches here is one the periodic task runs. + if config.periodic_script_bash.is_some() { + tracing::warn!( + worker = %worker_name, hostname = %hostname, + "EXIT_AFTER_N_JOBS is set and this worker group has a periodic script: it runs \ + once when the worker starts, so it runs every {max_jobs} job(s) whatever its \ + interval says." + ); + } } #[cfg(feature = "benchmark")] @@ -3066,7 +3152,26 @@ pub async fn run_worker( otel_set_worker_uptime(&worker_name, start_time.elapsed().as_secs_f64()); - if last_ping.elapsed().as_secs() > NUM_SECS_PING { + // The external IP resolves in the background, after the initial ping. Pinging on the very + // next iteration rather than the next periodic one is what gets it into the row of a worker + // whose process is short-lived (EXIT_AFTER_N_JOBS). + let ip = cached_ip(); + let ip_just_resolved = reported_ip.is_none() && ip.is_some(); + if ip_just_resolved || last_ping.elapsed().as_secs() > NUM_SECS_PING { + // Servers older than the background lookup take an IP from the initial ping only, so an + // agent has to register a second time to deliver whatever the lookup settled on, an + // address or the unretrievable marker. Registering also clears the row's job columns, + // which costs at most the last job's id here: no job of this worker is in flight at this + // point in the loop, and the next one refills them. + if ip_just_resolved && conn.as_sql().is_none() { + if let Err(e) = insert_ping(hostname, &worker_name, ip, &conn).await { + tracing::warn!( + worker = %worker_name, hostname = %hostname, + "failed to re-register with the resolved external IP: {e}" + ); + } + } + let read_cgroups = *REFRESH_CGROUP_READINGS && last_reading.elapsed().as_secs() > NUM_SECS_READINGS; update_worker_ping_full( @@ -3077,6 +3182,7 @@ pub async fn run_worker( &hostname, &mut occupancy_metrics, &killpill_tx, + ip, ) .await; @@ -3084,6 +3190,7 @@ pub async fn run_worker( last_reading = Instant::now(); } last_ping = Instant::now(); + reported_ip = ip; } if (jobs_executed as u32 + vacuum_shift) % VACUUM_PERIOD == 0 { diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index 7066b35945..42246c6358 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -2923,15 +2923,39 @@ pub async fn handle_flow( // its own disable write failed. Retry it: rearm_schedule turns // these into NoOp, so without disabling here the schedule would // stay enabled yet never run. - if let Err(disable_err) = sqlx::query!( - "UPDATE schedule SET enabled = false, error = $1 WHERE workspace_id = $2 AND path = $3", - err.to_string(), - &flow_job.workspace_id, - &schedule.path - ) - .execute(db) - .await - { + // Contract on `record_in_disable_tx`. Worth knowing here: + // this is the last chance to disable, because a flow + // schedule arms its next occurrence when the flow *starts*, + // so once the flow is gone nothing reaches this code again. + let mut history_lost = None; + let disable_result = async { + let mut tx = db.begin().await?; + let rows = sqlx::query!( + "UPDATE schedule SET enabled = false, error = $1 WHERE workspace_id = $2 AND path = $3 AND enabled = true", + err.to_string(), + &flow_job.workspace_id, + &schedule.path + ) + .execute(&mut *tx) + .await? + .rows_affected(); + if rows > 0 { + history_lost = windmill_common::trigger_history::record_in_disable_tx( + &mut tx, + windmill_queue::jobs::schedule_auto_disable_event( + &flow_job.workspace_id, + &schedule.path, + &err.to_string(), + ), + ) + .await; + } + tx.commit().await?; + Ok::<(), Error>(()) + } + .await; + + if let Err(disable_err) = disable_result { report_error_to_workspace_handler_or_critical_side_channel( &mini_job, db, @@ -2942,6 +2966,17 @@ pub async fn handle_flow( ) .await; } + if let Some(history_err) = history_lost { + report_error_to_workspace_handler_or_critical_side_channel( + &mini_job, + db, + format!( + "Disabled schedule {} but could not record it in the trigger history: {history_err}", + schedule.path, + ), + ) + .await; + } } else { // Transient error (DB contention, timeout) after retry exhaustion: // not the schedule's fault. Report it but leave the schedule diff --git a/backend/windmill-worker/src/worker_utils.rs b/backend/windmill-worker/src/worker_utils.rs index 6f47b2d863..67e9831481 100644 --- a/backend/windmill-worker/src/worker_utils.rs +++ b/backend/windmill-worker/src/worker_utils.rs @@ -4,6 +4,7 @@ use uuid::Uuid; use windmill_common::{ agent_workers::{PingJobStatus, PingJobStatusResponse}, cache, + external_ip::UNKNOWN_IP, worker::{ get_memory, get_vcpus, get_windmill_memory_usage, get_worker_memory_usage, insert_ping_query, update_job_ping_query, update_worker_ping_from_job_query, @@ -26,6 +27,7 @@ pub(crate) async fn update_worker_ping_full( hostname: &str, occupancy_metrics: &mut OccupancyMetrics, killpill_tx: &KillpillSender, + ip: Option<&str>, ) { let wc = WORKER_CONFIG.load(); let tags = wc.worker_tags.clone(); @@ -64,6 +66,7 @@ pub(crate) async fn update_worker_ping_full( occupancy_rate_5m, occupancy_rate_30m, native_mode, + ip, ) }) .retry( @@ -110,6 +113,7 @@ async fn update_worker_ping_full_inner( occupancy_rate_5m: Option, occupancy_rate_30m: Option, native_mode: bool, + ip: Option<&str>, ) -> anyhow::Result<()> { match conn { Connection::Sql(db) => { @@ -126,6 +130,7 @@ async fn update_worker_ping_full_inner( occupancy_rate_5m, occupancy_rate_30m, native_mode, + ip, db, ) .await?; @@ -139,7 +144,7 @@ async fn update_worker_ping_full_inner( last_job_executed: None, last_job_workspace_id: None, worker_instance: None, - ip: None, + ip: ip.map(str::to_string), tags: Some(tags.to_vec()), dw: None, dws: None, @@ -169,7 +174,7 @@ async fn update_worker_ping_full_inner( pub async fn insert_ping( worker_instance: &str, worker_name: &str, - ip: &str, + ip: Option<&str>, db: &Connection, ) -> anyhow::Result { let (tags, dw, dws, native_mode) = { @@ -228,7 +233,10 @@ pub async fn insert_ping( last_job_executed: None, last_job_workspace_id: None, worker_instance: Some(worker_instance.to_string()), - ip: Some(ip.to_string()), + // Servers older than the background lookup reject an initial ping with + // no IP, and an agent worker routinely runs against one, so the + // not-resolved-yet case goes over the wire as the sentinel. + ip: Some(ip.unwrap_or(UNKNOWN_IP).to_string()), tags: Some(tags.to_vec()), dw: dw, dws: dws, diff --git a/cli/src/core/client.ts b/cli/src/core/client.ts index f7073fe501..32776bd498 100644 --- a/cli/src/core/client.ts +++ b/cli/src/core/client.ts @@ -14,6 +14,17 @@ export function markRequestsAsSyncOrigin() { OpenAPI.HEADERS = { ...existing, "X-Windmill-Deploy-Origin": "sync" }; } +/** + * Name this process as the CLI on every subsequent request, so a trigger the + * CLI created or disabled is attributed to `cli` rather than to a bare API + * call in `trigger_history`. Attribution only — nothing on the server grants + * anything on the strength of it. + */ +export function markRequestsAsCliClient() { + const existing = typeof OpenAPI.HEADERS === "object" ? OpenAPI.HEADERS : {}; + OpenAPI.HEADERS = { ...existing, "X-Windmill-Client": "cli" }; +} + export function setClient(token?: string, baseUrl?: string) { if (baseUrl === undefined) { baseUrl = process.env["BASE_INTERNAL_URL"] ?? diff --git a/cli/src/main.ts b/cli/src/main.ts index ff8c576ee9..ccce20da30 100755 --- a/cli/src/main.ts +++ b/cli/src/main.ts @@ -32,6 +32,7 @@ import { OpenAPI } from "../gen/index.ts"; import { getHeaders } from "./utils/utils.ts"; import { detectAuthGatewayChallenge } from "./utils/http_guards.ts"; import { setShowDiffs } from "./core/conf.ts"; +import { markRequestsAsCliClient } from "./core/client.ts"; import { NpmProvider } from "./utils/upgrade.ts"; import { pull as hubPull } from "./commands/hub/hub.ts"; import { pull, push } from "./commands/sync/sync.ts"; @@ -300,6 +301,7 @@ async function main() { if (extraHeaders) { OpenAPI.HEADERS = extraHeaders; } + markRequestsAsCliClient(); OpenAPI.interceptors.response.use(async (response) => { await detectAuthGatewayChallenge(response); return response; diff --git a/docs/feature-telemetry.md b/docs/feature-telemetry.md index fed78cc064..947329fc45 100644 --- a/docs/feature-telemetry.md +++ b/docs/feature-telemetry.md @@ -4,9 +4,9 @@ anonymous usage-stats payload. It answers "does anyone use this, and which variant do they pick" without any identifying data leaving the instance. -It currently carries 14 registered actions across three features (`ai_session`, `ai_chat`, -`flow_editor`). Nearly all of the product is uninstrumented, so new user-facing work is the -opportunity to change that. +It currently carries 20 registered actions across eight features (`ai_session`, `ai_chat`, +`flow_editor`, `flow_run`, `flow_step`, `trigger`, `command_script`, `hub_script`). Nearly all of +the product is uninstrumented, so new user-facing work is the opportunity to change that. ## When to instrument @@ -49,9 +49,10 @@ vocabulary closed and small — enumerate the values in a TS union next to the c Four steps. Skipping step 1 or 3 fails quietly. **1. Register the pair** in `FEATURE_USAGE_KINDS` -(`backend/windmill-api-workspaces/src/workspaces.rs`). An unregistered `(feature, kind)` is -dropped by `valid_feature_usage_event` with a bare `continue` — no error, no log, still a 204 to -the browser. Frontend-only instrumentation records **nothing** and looks like it worked. +(`backend/windmill-common/src/feature_usage_ee.rs`, tracked in `windmill-ee-private`). An +unregistered `(feature, kind)` is dropped by `is_recordable_event` with a bare `continue` — no +error, no log, still a 204 to the browser. Frontend-only instrumentation records **nothing** and +looks like it worked. **2. Log from the frontend:** @@ -75,6 +76,10 @@ under-discloses what it sends. This has already drifted once. SELECT feature, kind, key, entity_id, day, value FROM feature_usage ORDER BY updated_at DESC LIMIT 10; ``` +Collection sits behind the `private` feature, so a public build records nothing from either the +HTTP route or the Rust helper. Run the backend with `--features enterprise,private` or this query +stays empty however correct the instrumentation is. + ## Privacy rules Only aggregated counts ever leave the instance, and only when telemetry is enabled and minimal @@ -85,8 +90,21 @@ cannot be collected — drop it. Counters aggregate over the last 30 days; rows are pruned after 60. -## Backend-only features +## Logging from the backend -Ingestion is frontend-only: `log_feature_usage` is an HTTP route the browser posts to, and there -is no Rust-side helper. A feature with no UI cannot be instrumented today without adding one. -Scope the default to user-facing work, and say so rather than implying backend coverage exists. +A feature with no UI is instrumented the same way, from Rust: + +```rust +windmill_common::feature_usage::log_feature_usage("trigger", "fired", kind.as_str()); +``` + +Same registry, same key rules, and the same silent drop when the pair is unregistered. `feature` +and `kind` are `&'static str` so a call site cannot pass a computed pair. The call increments an +in-memory counter and returns; the monitor loop flushes the accumulator, so it is cheap enough for +hot paths — but only cheap per call, not free: a key with unbounded cardinality would grow the map +until it hits the per-action cap and starts dropping new keys. + +There is no `entity_id` and no explicit `value` on this path: it counts occurrences. + +`feature_usage_ee` holds the registry and the writer; the public build gets the inert +`feature_usage_oss`, since a CE instance never sends a stats payload. diff --git a/docs/validation.md b/docs/validation.md index 3bfd1325ce..050eacd75b 100644 --- a/docs/validation.md +++ b/docs/validation.md @@ -39,6 +39,7 @@ After all code changes are done, run `./update_sqlx.sh` from `backend/` to regen | Modified Flow structures | Also update `openflow.openapi.yaml` | | Changed DB schema | Update `backend/summarized_schema.txt` if needed | | Enterprise file changes | Companion PR in `windmill-ee-private` (see `docs/enterprise.md`) | +| Changed a hook in `.claude/hooks/` | `bash .claude/hooks/test-hooks.sh` — pins which commands prompt | ## When to Write Tests diff --git a/frontend/src/lib/components/AppConnectDrawer.svelte b/frontend/src/lib/components/AppConnectDrawer.svelte index 39df15721e..a84877e39b 100644 --- a/frontend/src/lib/components/AppConnectDrawer.svelte +++ b/frontend/src/lib/components/AppConnectDrawer.svelte @@ -14,7 +14,11 @@ disableChatOffset?: boolean } - let { expressOAuthSetup = false, workspace = undefined, disableChatOffset = false }: Props = $props() + let { + expressOAuthSetup = false, + workspace = undefined, + disableChatOffset = false + }: Props = $props() let drawer: Drawer | undefined = $state() let resourceType = $state('') diff --git a/frontend/src/lib/components/AppConnectInner.svelte b/frontend/src/lib/components/AppConnectInner.svelte index dad835c7e7..e163c2ad07 100644 --- a/frontend/src/lib/components/AppConnectInner.svelte +++ b/frontend/src/lib/components/AppConnectInner.svelte @@ -561,7 +561,9 @@ args = {} } else { getResourceTypeInfo() - getScopesAndParams() + // Awaited: the popup is built from `scopes`, so advancing before this + // resolves sends the user to an authorize url with no scope at all. + await getScopesAndParams() } step += 1 } else if (step == 2 && !manual) { diff --git a/frontend/src/lib/components/AuthSettings.svelte b/frontend/src/lib/components/AuthSettings.svelte index 2ad6ab917a..c5f0a2040a 100644 --- a/frontend/src/lib/components/AuthSettings.svelte +++ b/frontend/src/lib/components/AuthSettings.svelte @@ -53,6 +53,13 @@ hideTabs = false }: Props = $props() + // The callback lands on a frontend route, so a base url that is not the origin + // the admin is browsing is almost always a misconfiguration. + let browserOrigin = typeof window !== 'undefined' ? window.location.origin : '' + let baseUrlMismatch = $derived( + !!baseUrl && !!browserOrigin && baseUrl.replace(/\/$/, '') !== browserOrigin + ) + $effect(() => { if (oauths == undefined) { oauths = {} @@ -522,6 +529,27 @@ bind:password={oauths[k]['secret']} /> +
+ Redirect URL + {#if !baseUrl} + + Set it in Core settings. The redirect url is built from it, and {k} needs the exact + value. + + {:else} + + {/if} + {#if baseUrlMismatch} + + This is built from the instance base url. Update it in Core settings if it is + wrong, or {k} will reject the callback. + + {/if} +
These credentials are for {#if !windmillBuiltins.includes(k) || (registryCcCapable(k) && registryAuthCodeCapable(k))} diff --git a/frontend/src/lib/components/BatchLoadProgress.svelte b/frontend/src/lib/components/BatchLoadProgress.svelte new file mode 100644 index 0000000000..16cf2d7212 --- /dev/null +++ b/frontend/src/lib/components/BatchLoadProgress.svelte @@ -0,0 +1,62 @@ + + +
+ Loading {itemsLabel}: {loaded} of {total}... +
+
+
+ {#if batchSize != null} + Batch size: + { + const v = parseInt(e.currentTarget.value) + if (v >= 1 && v <= maxBatchSize) { + onBatchSizeChange?.(v) + } else { + e.currentTarget.value = String(batchSize) + } + } + }} + /> + {/if} + +
diff --git a/frontend/src/lib/components/DropdownSubmenuItem.svelte b/frontend/src/lib/components/DropdownSubmenuItem.svelte index ed08fd7abd..631f6374bd 100644 --- a/frontend/src/lib/components/DropdownSubmenuItem.svelte +++ b/frontend/src/lib/components/DropdownSubmenuItem.svelte @@ -3,7 +3,8 @@ import MenuItem from '$lib/components/meltComponents/MenuItem.svelte' import { melt } from '@melt-ui/svelte' import { twMerge } from 'tailwind-merge' - import { ChevronRight } from 'lucide-svelte' + import { Check, ChevronRight } from 'lucide-svelte' + import Toggle from '$lib/components/Toggle.svelte' import type { Item } from '$lib/utils' import type { MenubarMenuElements, createDropdownMenu } from '@melt-ui/svelte' import { Tooltip } from './meltComponents' @@ -65,12 +66,29 @@ item={meltItem} > {#if subItem.icon} - + {/if}

{subItem.displayName}

{@render subItem.extra?.()} + {#if subItem.shortcut || subItem.selected || subItem.toggle !== undefined} +
+ {#if subItem.shortcut} + {subItem.shortcut} + {/if} + {#if subItem.selected} + + {/if} + {#if subItem.toggle !== undefined} + + + {/if} +
+ {/if} {#if subItem.tooltip} {#snippet text()} diff --git a/frontend/src/lib/components/DropdownV2Inner.svelte b/frontend/src/lib/components/DropdownV2Inner.svelte index 1196589474..425c41d8f5 100644 --- a/frontend/src/lib/components/DropdownV2Inner.svelte +++ b/frontend/src/lib/components/DropdownV2Inner.svelte @@ -1,5 +1,6 @@ @@ -476,7 +422,9 @@
+ {#if batchProgress} +
+ onBatchSizeChange?.(size)} + onStop={() => onStopLoading?.()} + /> +
+ {/if}
Per page: - - +
{#if status === 'idle'} - + {#if noOAuth} +
{server.name} did not advertise OAuth support.
+ {/if} + {:else if status === 'discovering'} -
Discovering OAuth settings...
+
Checking what {server.name} supports...
{:else if status === 'discovered' && discoveryResult} -
- ✓ OAuth supported - {#if discoveryResult.supports_dynamic_registration} - (Dynamic Client Registration available) - {/if} -
- {#if discoveryResult.scopes_supported && discoveryResult.scopes_supported.length > 0} -